Examples with SerializationFactory org.apache.hadoop.io.serializer.SerializationFactory used on opensource projects

Example 16 with SerializationFactory

use of org.apache.hadoop.io.serializer.SerializationFactory in project hadoop by apache.

the class ReflectionUtils method copy.

/**
   * Make a copy of the writable object using serialization to a buffer
   * @param src the object to copy from
   * @param dst the object to copy into, which is destroyed
   * @return dst param (the copy)
   * @throws IOException
   */
@SuppressWarnings("unchecked")
public static <T> T copy(Configuration conf, T src, T dst) throws IOException {
    CopyInCopyOutBuffer buffer = CLONE_BUFFERS.get();
    buffer.outBuffer.reset();
    SerializationFactory factory = getFactory(conf);
    Class<T> cls = (Class<T>) src.getClass();
    Serializer<T> serializer = factory.getSerializer(cls);
    serializer.open(buffer.outBuffer);
    serializer.serialize(src);
    buffer.moveData();
    Deserializer<T> deserializer = factory.getDeserializer(cls);
    deserializer.open(buffer.inBuffer);
    dst = deserializer.deserialize(dst);
    return dst;
}

Example 17 with SerializationFactory

use of org.apache.hadoop.io.serializer.SerializationFactory in project hadoop by apache.

the class JobSplitWriter method writeNewSplits.

@SuppressWarnings("unchecked")
private static <T extends InputSplit> SplitMetaInfo[] writeNewSplits(Configuration conf, T[] array, FSDataOutputStream out) throws IOException, InterruptedException {
    SplitMetaInfo[] info = new SplitMetaInfo[array.length];
    if (array.length != 0) {
        SerializationFactory factory = new SerializationFactory(conf);
        int i = 0;
        int maxBlockLocations = conf.getInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY, MRConfig.MAX_BLOCK_LOCATIONS_DEFAULT);
        long offset = out.getPos();
        for (T split : array) {
            long prevCount = out.getPos();
            Text.writeString(out, split.getClass().getName());
            Serializer<T> serializer = factory.getSerializer((Class<T>) split.getClass());
            serializer.open(out);
            serializer.serialize(split);
            long currCount = out.getPos();
            String[] locations = split.getLocations();
            if (locations.length > maxBlockLocations) {
                LOG.warn("Max block location exceeded for split: " + split + " splitsize: " + locations.length + " maxsize: " + maxBlockLocations);
                locations = Arrays.copyOf(locations, maxBlockLocations);
            }
            info[i++] = new JobSplit.SplitMetaInfo(locations, offset, split.getLength());
            offset += currCount - prevCount;
        }
    }
    return info;
}

Example 18 with SerializationFactory

use of org.apache.hadoop.io.serializer.SerializationFactory in project gora by apache.

the class IOUtils method serialize.

/**
   * Serializes the object to the given data output using
   * available Hadoop serializations.
   *
   * @param conf Hadoop conf.
   * @param obj object instance to be serialized.
   * @param out data stream which serialized content is written.
   * @param objClass Class type of the object to be serialized.
   * @param <T> class type of object to be serialized.
   * @throws IOException occurred while serializing the object to bytes.
   */
public static <T> void serialize(Configuration conf, DataOutput out, T obj, Class<T> objClass) throws IOException {
    SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
    Serializer<T> serializer = serializationFactory.getSerializer(objClass);
    try (ByteBufferOutputStream os = new ByteBufferOutputStream()) {
        serializer.open(os);
        serializer.serialize(obj);
        int length = 0;
        List<ByteBuffer> buffers = os.getBufferList();
        for (ByteBuffer buffer : buffers) {
            length += buffer.limit() - buffer.arrayOffset();
        }
        WritableUtils.writeVInt(out, length);
        for (ByteBuffer buffer : buffers) {
            byte[] arr = buffer.array();
            out.write(arr, buffer.arrayOffset(), buffer.limit());
        }
    } finally {
        if (serializer != null)
            serializer.close();
    }
}

Example 19 with SerializationFactory

use of org.apache.hadoop.io.serializer.SerializationFactory in project ignite by apache.

the class HadoopV2TaskContext method readExternalSplit.

/**
     * @param split External split.
     * @return Native input split.
     * @throws IgniteCheckedException If failed.
     */
@SuppressWarnings("unchecked")
private Object readExternalSplit(HadoopExternalSplit split) throws IgniteCheckedException {
    Path jobDir = new Path(jobConf().get(MRJobConfig.MAPREDUCE_JOB_DIR));
    FileSystem fs;
    try {
        fs = fileSystemForMrUserWithCaching(jobDir.toUri(), jobConf(), fsMap);
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
    try (FSDataInputStream in = fs.open(JobSubmissionFiles.getJobSplitFile(jobDir))) {
        in.seek(split.offset());
        String clsName = Text.readString(in);
        Class<?> cls = jobConf().getClassByName(clsName);
        assert cls != null;
        Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls);
        Deserializer deserializer = serialization.getDeserializer(cls);
        deserializer.open(in);
        Object res = deserializer.deserialize(null);
        deserializer.close();
        assert res != null;
        return res;
    } catch (IOException | ClassNotFoundException e) {
        throw new IgniteCheckedException(e);
    }
}

Example 20 with SerializationFactory

use of org.apache.hadoop.io.serializer.SerializationFactory in project ignite by apache.

the class HadoopV2TaskContext method getSerialization.

/**
     * Gets serializer for specified class.
     *
     * @param cls Class.
     * @param jobConf Job configuration.
     * @return Appropriate serializer.
     */
@SuppressWarnings("unchecked")
private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf) throws IgniteCheckedException {
    A.notNull(cls, "cls");
    SerializationFactory factory = new SerializationFactory(jobConf);
    Serialization<?> serialization = factory.getSerialization(cls);
    if (serialization == null)
        throw new IgniteCheckedException("Failed to find serialization for: " + cls.getName());
    if (serialization.getClass() == WritableSerialization.class)
        return new HadoopWritableSerialization((Class<? extends Writable>) cls);
    return new HadoopSerializationWrapper(serialization, cls);
}