Examples with ByteBufferInputStream org.apache.avro.util.ByteBufferInputStream used on opensource projects

Example 1 with ByteBufferInputStream

use of org.apache.avro.util.ByteBufferInputStream in project druid by druid-io.

the class InlineSchemaAvroBytesDecoder method parse.

@Override
public GenericRecord parse(ByteBuffer bytes) {
    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schemaObj);
    ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes));
    try {
        return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    } catch (Exception e) {
        throw new ParseException(e, "Fail to decode avro message!");
    }
}

Example 2 with ByteBufferInputStream

use of org.apache.avro.util.ByteBufferInputStream in project druid by druid-io.

the class InlineSchemasAvroBytesDecoder method parse.

// It is assumed that record has following format.
// byte 1 : version, static 0x1
// byte 2-5 : int schemaId
// remaining bytes would have avro data
@Override
public GenericRecord parse(ByteBuffer bytes) {
    if (bytes.remaining() < 5) {
        throw new ParseException("record must have at least 5 bytes carrying version and schemaId");
    }
    byte version = bytes.get();
    if (version != V1) {
        throw new ParseException("found record of arbitrary version [%s]", version);
    }
    int schemaId = bytes.getInt();
    Schema schemaObj = schemaObjs.get(schemaId);
    if (schemaObj == null) {
        throw new ParseException("Failed to find schema for id [%s]", schemaId);
    }
    try {
        DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schemaObj);
        ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes));
        return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    } catch (Exception e) {
        throw new ParseException(e, "Fail to decode avro message with schemaId [%s].", schemaId);
    }
}

Example 3 with ByteBufferInputStream

use of org.apache.avro.util.ByteBufferInputStream in project gora by apache.

the class IOUtils method deserialize.

/**
   * Deserializes the object in the given data input using
   * available Hadoop serializations.
   *
   * @param conf Hadoop conf.
   * @param in data input stream where serialized content is read.
   * @param <T> object class type.
   * @param obj data object.
   * @param objClass object class type.
   * @throws IOException occurred while deserializing the byte content.
   * @return deserialized object.
   */
public static <T> T deserialize(Configuration conf, DataInput in, T obj, Class<T> objClass) throws IOException {
    SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
    Deserializer<T> deserializer = serializationFactory.getDeserializer(objClass);
    int length = WritableUtils.readVInt(in);
    byte[] arr = new byte[length];
    in.readFully(arr);
    List<ByteBuffer> list = new ArrayList<>();
    list.add(ByteBuffer.wrap(arr));
    try (ByteBufferInputStream is = new ByteBufferInputStream(list)) {
        deserializer.open(is);
        T newObj = deserializer.deserialize(obj);
        return newObj;
    } finally {
        if (deserializer != null)
            deserializer.close();
    }
}

Example 4 with ByteBufferInputStream

use of org.apache.avro.util.ByteBufferInputStream in project druid by druid-io.

the class SchemaRepoBasedAvroBytesDecoder method parse.

@Override
public GenericRecord parse(ByteBuffer bytes) {
    Pair<SUBJECT, ID> subjectAndId = subjectAndIdConverter.getSubjectAndId(bytes);
    Schema schema = typedRepository.getSchema(subjectAndId.lhs, subjectAndId.rhs);
    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
    ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes));
    try {
        return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    } catch (IOException e) {
        throw new ParseException(e, "Fail to decode avro message!");
    }
}

Example 5 with ByteBufferInputStream

use of org.apache.avro.util.ByteBufferInputStream in project gora by apache.

the class TestIOUtils method testSerializeDeserialize.

@SafeVarargs
@SuppressWarnings("unchecked")
public static <T> void testSerializeDeserialize(T... objects) throws Exception {
    ByteBufferOutputStream os = new ByteBufferOutputStream();
    DataOutputStream dos = new DataOutputStream(os);
    ByteBufferInputStream is = null;
    DataInputStream dis = null;
    GoraMapReduceUtils.setIOSerializations(conf, true);
    try {
        for (T before : objects) {
            IOUtils.serialize(conf, dos, before, (Class<T>) before.getClass());
            dos.flush();
        }
        is = new ByteBufferInputStream(os.getBufferList());
        dis = new DataInputStream(is);
        for (T before : objects) {
            T after = IOUtils.deserialize(conf, dis, null, (Class<T>) before.getClass());
            if (before instanceof BoolArrayWrapper) {
                if (after instanceof BoolArrayWrapper) {
                    log.info("Before : " + java.util.Arrays.toString(((BoolArrayWrapper) before).arr));
                    log.info("After : " + java.util.Arrays.toString(((BoolArrayWrapper) after).arr));
                }
            } else if (before instanceof StringArrayWrapper) {
                if (after instanceof StringArrayWrapper) {
                    log.info("Before : " + java.util.Arrays.toString(((StringArrayWrapper) before).arr));
                    log.info("After : " + java.util.Arrays.toString(((StringArrayWrapper) after).arr));
                }
            } else {
                log.info("Before : " + before);
                log.info("After : " + before);
            }
            assertEquals(before, after);
            if ((before instanceof PersistentBase) && (after instanceof PersistentBase)) {
                assertEquals(Arrays.equals(((PersistentBase) before).getDirtyBytes().array(), ((PersistentBase) after).getDirtyBytes().array()), true);
            }
        }
        //assert that the end of input is reached
        long skipped = dis.skip(1);
        assertEquals(0, skipped);
    } catch (EOFException expected) {
    //either should throw exception or return 0 as skipped
    } finally {
        org.apache.hadoop.io.IOUtils.closeStream(dos);
        org.apache.hadoop.io.IOUtils.closeStream(os);
        org.apache.hadoop.io.IOUtils.closeStream(dis);
        org.apache.hadoop.io.IOUtils.closeStream(is);
    }
}