Examples with GenericRecordBuilder org.apache.avro.generic.GenericRecordBuilder used on opensource projects

Example 41 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project JavaBigData by TALKDATA.

the class TestAvroEventSerializer method createAvroFile.

public void createAvroFile(File file, String codec, boolean useSchemaUrl, boolean useStaticSchemaUrl) throws IOException {
    // serialize a few events using the reflection-based avro serializer
    OutputStream out = new FileOutputStream(file);
    Context ctx = new Context();
    if (codec != null) {
        ctx.put("compressionCodec", codec);
    }
    Schema schema = Schema.createRecord("myrecord", null, null, false);
    schema.setFields(Arrays.asList(new Schema.Field[] { new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null) }));
    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
    File schemaFile = null;
    if (useSchemaUrl || useStaticSchemaUrl) {
        schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
        Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
    }
    if (useStaticSchemaUrl) {
        ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL, schemaFile.toURI().toURL().toExternalForm());
    }
    EventSerializer.Builder builder = new AvroEventSerializer.Builder();
    EventSerializer serializer = builder.build(ctx, out);
    serializer.afterCreate();
    for (int i = 0; i < 3; i++) {
        GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
        Event event = EventBuilder.withBody(serializeAvro(record, schema));
        if (schemaFile == null && !useSchemaUrl) {
            event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER, schema.toString());
        } else if (useSchemaUrl) {
            event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER, schemaFile.toURI().toURL().toExternalForm());
        }
        serializer.write(event);
    }
    serializer.flush();
    serializer.beforeClose();
    out.flush();
    out.close();
    if (schemaFile != null) {
        schemaFile.delete();
    }
}

Example 42 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project avro by a0x8o.

the class TestSchemaBuilder method testDefaults.

@Test
public void testDefaults() throws IOException {
    Schema writeSchema = SchemaBuilder.record("r").fields().name("requiredInt").type().intType().noDefault().name("optionalInt").type().optional().intType().name("nullableIntWithDefault").type().nullable().intType().intDefault(3).endRecord();
    GenericData.Record rec1 = new GenericRecordBuilder(writeSchema).set("requiredInt", 1).build();
    Assert.assertEquals(1, rec1.get("requiredInt"));
    Assert.assertEquals(null, rec1.get("optionalInt"));
    Assert.assertEquals(3, rec1.get("nullableIntWithDefault"));
    GenericData.Record rec2 = new GenericRecordBuilder(writeSchema).set("requiredInt", 1).set("optionalInt", 2).set("nullableIntWithDefault", 13).build();
    Assert.assertEquals(1, rec2.get("requiredInt"));
    Assert.assertEquals(2, rec2.get("optionalInt"));
    Assert.assertEquals(13, rec2.get("nullableIntWithDefault"));
    // write to file
    File file = new File(DIR.getRoot().getPath(), "testDefaults.avro");
    try (DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>())) {
        writer.create(writeSchema, file);
        writer.append(rec1);
        writer.append(rec2);
    }
    Schema readSchema = SchemaBuilder.record("r").fields().name("requiredInt").type().intType().noDefault().name("optionalInt").type().optional().intType().name("nullableIntWithDefault").type().nullable().intType().intDefault(3).name("newOptionalInt").type().optional().intType().name("newNullableIntWithDefault").type().nullable().intType().intDefault(5).endRecord();
    try (DataFileReader<GenericData.Record> reader = new DataFileReader<>(file, new GenericDatumReader<>(writeSchema, readSchema))) {
        GenericData.Record rec1read = reader.iterator().next();
        Assert.assertEquals(1, rec1read.get("requiredInt"));
        Assert.assertNull(rec1read.get("optionalInt"));
        Assert.assertEquals(3, rec1read.get("nullableIntWithDefault"));
        Assert.assertNull(rec1read.get("newOptionalInt"));
        Assert.assertEquals(5, rec1read.get("newNullableIntWithDefault"));
        GenericData.Record rec2read = reader.iterator().next();
        Assert.assertEquals(1, rec2read.get("requiredInt"));
        Assert.assertEquals(2, rec2read.get("optionalInt"));
        Assert.assertEquals(13, rec2read.get("nullableIntWithDefault"));
        Assert.assertNull(rec2read.get("newOptionalInt"));
        Assert.assertEquals(5, rec2read.get("newNullableIntWithDefault"));
    }
}

Example 43 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project avro by a0x8o.

the class TestResolvingGrammarGenerator method testDifferingEnumNamespaces.

@Test
public void testDifferingEnumNamespaces() throws Exception {
    Schema schema1 = SchemaBuilder.record("MyRecord").fields().name("field").type(ENUM1_AB_SCHEMA_NAMESPACE_1).noDefault().endRecord();
    Schema schema2 = SchemaBuilder.record("MyRecord").fields().name("field").type(ENUM1_AB_SCHEMA_NAMESPACE_2).noDefault().endRecord();
    GenericData.EnumSymbol genericEnumSymbol = new GenericData.EnumSymbol(ENUM1_AB_SCHEMA_NAMESPACE_1, "A");
    GenericData.Record record = new GenericRecordBuilder(schema1).set("field", genericEnumSymbol).build();
    byte[] data = writeRecord(schema1, record);
    Assert.assertEquals(genericEnumSymbol, readRecord(schema1, data).get("field"));
    Assert.assertEquals(genericEnumSymbol, readRecord(schema2, data).get("field"));
}

Example 44 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project flume by apache.

the class TestLog4jAppenderWithAvro method testDifferentEventTypesInBatchWithAvroGeneric.

@Test
public void testDifferentEventTypesInBatchWithAvroGeneric() throws IOException {
    loadProperties("flume-log4jtest-avro-generic.properties");
    props.put("log4j.appender.out2.Port", String.valueOf(port));
    PropertyConfigurator.configure(props);
    Logger logger = LogManager.getLogger(getClass());
    String msg = "Avro log message";
    Schema schema = new Schema.Parser().parse(getClass().getClassLoader().getResource("myrecord.avsc").openStream());
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    GenericRecord record = builder.set("message", msg).build();
    List<Object> events = Arrays.asList("string", record);
    logger.info(events);
    Transaction transaction = ch.getTransaction();
    transaction.begin();
    Event event = ch.take();
    Assert.assertNotNull(event);
    Assert.assertEquals("string", new String(event.getBody()));
    event = ch.take();
    Assert.assertNotNull(event);
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
    GenericRecord recordFromEvent = reader.read(null, decoder);
    Assert.assertEquals(msg, recordFromEvent.get("message").toString());
    Map<String, String> hdrs = event.getHeaders();
    Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));
    Assert.assertEquals("Schema URL should be set", "file:///tmp/myrecord.avsc", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString()));
    Assert.assertNull("Schema string should not be set", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString()));
    transaction.commit();
    transaction.close();
}

Example 45 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project flume by apache.

the class TestLog4jAppenderWithAvro method testAvroGeneric.

@Test
public void testAvroGeneric() throws IOException {
    loadProperties("flume-log4jtest-avro-generic.properties");
    props.put("log4j.appender.out2.Port", String.valueOf(port));
    PropertyConfigurator.configure(props);
    Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class);
    String msg = "This is log message number " + String.valueOf(0);
    Schema schema = new Schema.Parser().parse(getClass().getClassLoader().getResource("myrecord.avsc").openStream());
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    GenericRecord record = builder.set("message", msg).build();
    logger.info(record);
    Transaction transaction = ch.getTransaction();
    transaction.begin();
    Event event = ch.take();
    Assert.assertNotNull(event);
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
    GenericRecord recordFromEvent = reader.read(null, decoder);
    Assert.assertEquals(msg, recordFromEvent.get("message").toString());
    Map<String, String> hdrs = event.getHeaders();
    Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));
    Assert.assertEquals("Schema URL should be set", "file:///tmp/myrecord.avsc", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString()));
    Assert.assertNull("Schema string should not be set", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString()));
    transaction.commit();
    transaction.close();
}