Examples with ByteBufferOutputStream org.apache.avro.util.ByteBufferOutputStream used on opensource projects

Example 1 with ByteBufferOutputStream

use of org.apache.avro.util.ByteBufferOutputStream in project gora by apache.

the class TestIOUtils method testSerializeDeserialize.

@SafeVarargs
@SuppressWarnings("unchecked")
public static <T> void testSerializeDeserialize(T... objects) throws Exception {
    ByteBufferOutputStream os = new ByteBufferOutputStream();
    DataOutputStream dos = new DataOutputStream(os);
    ByteBufferInputStream is = null;
    DataInputStream dis = null;
    GoraMapReduceUtils.setIOSerializations(conf, true);
    try {
        for (T before : objects) {
            IOUtils.serialize(conf, dos, before, (Class<T>) before.getClass());
            dos.flush();
        }
        is = new ByteBufferInputStream(os.getBufferList());
        dis = new DataInputStream(is);
        for (T before : objects) {
            T after = IOUtils.deserialize(conf, dis, null, (Class<T>) before.getClass());
            if (before instanceof BoolArrayWrapper) {
                if (after instanceof BoolArrayWrapper) {
                    log.info("Before : " + java.util.Arrays.toString(((BoolArrayWrapper) before).arr));
                    log.info("After : " + java.util.Arrays.toString(((BoolArrayWrapper) after).arr));
                }
            } else if (before instanceof StringArrayWrapper) {
                if (after instanceof StringArrayWrapper) {
                    log.info("Before : " + java.util.Arrays.toString(((StringArrayWrapper) before).arr));
                    log.info("After : " + java.util.Arrays.toString(((StringArrayWrapper) after).arr));
                }
            } else {
                log.info("Before : " + before);
                log.info("After : " + before);
            }
            assertEquals(before, after);
            if ((before instanceof PersistentBase) && (after instanceof PersistentBase)) {
                assertEquals(Arrays.equals(((PersistentBase) before).getDirtyBytes().array(), ((PersistentBase) after).getDirtyBytes().array()), true);
            }
        }
        //assert that the end of input is reached
        long skipped = dis.skip(1);
        assertEquals(0, skipped);
    } catch (EOFException expected) {
    //either should throw exception or return 0 as skipped
    } finally {
        org.apache.hadoop.io.IOUtils.closeStream(dos);
        org.apache.hadoop.io.IOUtils.closeStream(os);
        org.apache.hadoop.io.IOUtils.closeStream(dis);
        org.apache.hadoop.io.IOUtils.closeStream(is);
    }
}

Example 2 with ByteBufferOutputStream

use of org.apache.avro.util.ByteBufferOutputStream in project gora by apache.

the class IOUtils method serialize.

/**
   * Serializes the object to the given data output using
   * available Hadoop serializations.
   *
   * @param conf Hadoop conf.
   * @param obj object instance to be serialized.
   * @param out data stream which serialized content is written.
   * @param objClass Class type of the object to be serialized.
   * @param <T> class type of object to be serialized.
   * @throws IOException occurred while serializing the object to bytes.
   */
public static <T> void serialize(Configuration conf, DataOutput out, T obj, Class<T> objClass) throws IOException {
    SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
    Serializer<T> serializer = serializationFactory.getSerializer(objClass);
    try (ByteBufferOutputStream os = new ByteBufferOutputStream()) {
        serializer.open(os);
        serializer.serialize(obj);
        int length = 0;
        List<ByteBuffer> buffers = os.getBufferList();
        for (ByteBuffer buffer : buffers) {
            length += buffer.limit() - buffer.arrayOffset();
        }
        WritableUtils.writeVInt(out, length);
        for (ByteBuffer buffer : buffers) {
            byte[] arr = buffer.array();
            out.write(arr, buffer.arrayOffset(), buffer.limit());
        }
    } finally {
        if (serializer != null)
            serializer.close();
    }
}