Examples with GenericRecord org.apache.avro.generic.GenericRecord used on opensource projects

Example 26 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project druid by druid-io.

the class InlineSchemasAvroBytesDecoder method parse.

// It is assumed that record has following format.
// byte 1 : version, static 0x1
// byte 2-5 : int schemaId
// remaining bytes would have avro data
@Override
public GenericRecord parse(ByteBuffer bytes) {
    if (bytes.remaining() < 5) {
        throw new ParseException("record must have at least 5 bytes carrying version and schemaId");
    }
    byte version = bytes.get();
    if (version != V1) {
        throw new ParseException("found record of arbitrary version [%s]", version);
    }
    int schemaId = bytes.getInt();
    Schema schemaObj = schemaObjs.get(schemaId);
    if (schemaObj == null) {
        throw new ParseException("Failed to find schema for id [%s]", schemaId);
    }
    try {
        DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schemaObj);
        ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes));
        return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    } catch (Exception e) {
        throw new ParseException(e, "Fail to decode avro message with schemaId [%s].", schemaId);
    }
}

Example 27 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project druid by druid-io.

the class AvroHadoopInputRowParserTest method buildPigAvro.

private static GenericRecord buildPigAvro(GenericRecord datum, String inputStorage, String outputStorage) throws IOException {
    final File tmpDir = Files.createTempDir();
    FileReader<GenericRecord> reader = null;
    PigServer pigServer = null;
    try {
        // 0. write avro object into temp file.
        File someAvroDatumFile = new File(tmpDir, "someAvroDatum.avro");
        DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(new GenericDatumWriter<GenericRecord>());
        dataFileWriter.create(SomeAvroDatum.getClassSchema(), someAvroDatumFile);
        dataFileWriter.append(datum);
        dataFileWriter.close();
        // 1. read avro files into Pig
        pigServer = new PigServer(ExecType.LOCAL);
        pigServer.registerQuery(String.format("A = LOAD '%s' USING %s;", someAvroDatumFile, inputStorage));
        // 2. write new avro file using AvroStorage
        File outputDir = new File(tmpDir, "output");
        pigServer.store("A", String.valueOf(outputDir), outputStorage);
        // 3. read avro object from AvroStorage
        reader = DataFileReader.openReader(new File(outputDir, "part-m-00000.avro"), new GenericDatumReader<GenericRecord>());
        return reader.next();
    } finally {
        if (pigServer != null) {
            pigServer.shutdown();
        }
        Closeables.close(reader, true);
        FileUtils.deleteDirectory(tmpDir);
    }
}

Example 28 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project druid by druid-io.

the class InlineSchemaAvroBytesDecoderTest method testParse.

@Test
public void testParse() throws Exception {
    GenericRecord someAvroDatum = AvroStreamInputRowParserTest.buildSomeAvroDatum();
    Schema schema = SomeAvroDatum.getClassSchema();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
    GenericRecord actual = new InlineSchemaAvroBytesDecoder(schema).parse(ByteBuffer.wrap(out.toByteArray()));
    Assert.assertEquals(someAvroDatum.get("id"), actual.get("id"));
}

Example 29 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project druid by druid-io.

the class SchemaRegistryBasedAvroBytesDecoderTest method testParse.

@Test
public void testParse() throws Exception {
    // Given
    when(registry.getByID(eq(1234))).thenReturn(SomeAvroDatum.getClassSchema());
    GenericRecord someAvroDatum = AvroStreamInputRowParserTest.buildSomeAvroDatum();
    Schema schema = SomeAvroDatum.getClassSchema();
    byte[] bytes = getAvroDatum(schema, someAvroDatum);
    ByteBuffer bb = ByteBuffer.allocate(bytes.length + 5).put((byte) 0).putInt(1234).put(bytes);
    bb.rewind();
    // When
    GenericRecord actual = new SchemaRegistryBasedAvroBytesDecoder(registry).parse(bb);
    // Then
    Assert.assertEquals(someAvroDatum.get("id"), actual.get("id"));
}

Example 30 with GenericRecord

use of org.apache.avro.generic.GenericRecord in project druid by druid-io.

the class SchemaRegistryBasedAvroBytesDecoderTest method getAvroDatum.

byte[] getAvroDatum(Schema schema, GenericRecord someAvroDatum) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
    return out.toByteArray();
}