Examples with GenericDatumReader org.apache.avro.generic.GenericDatumReader used on opensource projects

Example 26 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project pinot by linkedin.

the class KafkaAvroMessageDecoder method decode.

@Override
public GenericRow decode(byte[] payload, int offset, int length, GenericRow destination) {
    if (payload == null || payload.length == 0 || length == 0) {
        return null;
    }
    byte[] md5 = Arrays.copyOfRange(payload, SCHEMA_HASH_START_OFFSET + offset, SCHEMA_HASH_END_OFFSET + offset);
    String md5String = hex(md5);
    org.apache.avro.Schema schema = null;
    boolean schemaUpdateFailed = false;
    if (md5ToAvroSchemaMap.containsKey(md5String)) {
        schema = md5ToAvroSchemaMap.get(md5String);
    } else {
        final String schemaUri = schemaRegistryBaseUrl + "/id=" + md5String;
        try {
            schema = fetchSchema(new URL(schemaUri));
            md5ToAvroSchemaMap.put(md5String, schema);
        } catch (Exception e) {
            schema = defaultAvroSchema;
            LOGGER.error("Error fetching schema using url {}. Attempting to continue with previous schema", schemaUri, e);
            schemaUpdateFailed = true;
        }
    }
    DatumReader<Record> reader = new GenericDatumReader<Record>(schema);
    try {
        GenericData.Record avroRecord = reader.read(null, decoderFactory.createBinaryDecoder(payload, HEADER_LENGTH + offset, length - HEADER_LENGTH, null));
        return avroRecordConvetrer.transform(avroRecord, schema, destination);
    } catch (IOException e) {
        LOGGER.error("Caught exception while reading message using schema {}{}", (schema == null ? "null" : schema.getName()), (schemaUpdateFailed ? "(possibly due to schema update failure)" : ""), e);
        return null;
    }
}

Example 27 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project rest.li by linkedin.

the class TestSchemaTranslator method genericRecordFromString.

public static GenericRecord genericRecordFromString(String jsonString, Schema writerSchema, Schema readerSchema) throws IOException {
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(writerSchema, readerSchema);
    byte[] bytes = jsonString.getBytes(Data.UTF_8_CHARSET);
    Decoder binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    GenericRecord record = reader.read(null, binaryDecoder);
    return record;
}

Example 28 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project rest.li by linkedin.

the class AvroUtil method genericRecordFromJson.

public static GenericRecord genericRecordFromJson(String json, Schema schema) throws IOException {
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
    AvroAdapter avroAdapter = AvroAdapterFinder.getAvroAdapter();
    Decoder jsonDecoder = avroAdapter.createJsonDecoder(schema, json);
    reader.setSchema(schema);
    GenericRecord record = reader.read(null, jsonDecoder);
    return record;
}

Example 29 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project rest.li by linkedin.

the class AvroUtil method genericRecordFromBytes.

public static GenericRecord genericRecordFromBytes(byte[] bytes, Schema schema) throws IOException {
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
    AvroAdapter avroAdapter = AvroAdapterFinder.getAvroAdapter();
    Decoder binaryDecoder = avroAdapter.createBinaryDecoder(bytes);
    reader.setSchema(schema);
    GenericRecord record = reader.read(null, binaryDecoder);
    return record;
}

Example 30 with GenericDatumReader

use of org.apache.avro.generic.GenericDatumReader in project databus by linkedin.

the class DbusEventAvroDecoder method dumpEventValueInJSON.

public void dumpEventValueInJSON(DbusEvent e, OutputStream out) {
    byte[] md5 = new byte[16];
    e.schemaId(md5);
    SchemaId schemaId = new SchemaId(md5);
    VersionedSchema sourceSchema = _schemaSet.getById(schemaId);
    ByteBuffer valueBuffer = e.value();
    byte[] valueBytes = new byte[valueBuffer.remaining()];
    valueBuffer.get(valueBytes);
    try {
        Schema schema = sourceSchema.getSchema();
        DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
        binDecoder.set(DecoderFactory.defaultFactory().createBinaryDecoder(valueBytes, binDecoder.get()));
        Object datum = reader.read(null, binDecoder.get());
        DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
        JsonGenerator g = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
        // write the src ID
        g.writeStartObject();
        g.writeFieldName(SRC_ID_FIELD_NAME);
        g.writeNumber(e.getSourceId());
        g.writeFieldName(OPCODE_FIELD_NAME);
        g.writeString(e.getOpcode().toString());
        g.writeFieldName("partId");
        g.writeNumber(Integer.valueOf(e.getPartitionId()));
        g.writeFieldName(VALUE_FIELD_NAME);
        writer.write(datum, new JsonEncoder(schema, g));
        g.writeEndObject();
        g.writeEndObject();
        try {
            g.writeEndObject();
        } catch (JsonGenerationException e_json) {
        // ignore the error as some how avro JsonEncoder may some times missing two }
        }
        g.flush();
    } catch (IOException e1) {
        LOG.error("event value serialization error; event = " + e, e1);
    }
}