Examples with GenericRecordBuilder org.apache.avro.generic.GenericRecordBuilder used on opensource projects

Example 61 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project hydrator-plugins by cdapio.

the class JoinerTestRun method verifyOuterJoinWithoutRequiredInputs.

private Void verifyOuterJoinWithoutRequiredInputs(Schema outputSchema, TimePartitionedFileSet fileSet) {
    try {
        Set<GenericRecord> actual = Sets.newHashSet(readOutput(fileSet, outputSchema));
        Assert.assertEquals(8, actual.size());
        org.apache.avro.Schema avroOutputSchema = new org.apache.avro.Schema.Parser().parse(outputSchema.toString());
        GenericRecord humtumRecord = new GenericRecordBuilder(avroOutputSchema).set("film_id", "4").set("film_name", "humtum").set("renamed_actor", null).set("renamed_category", null).build();
        GenericRecord sultanRecord = new GenericRecordBuilder(avroOutputSchema).set("film_id", null).set("film_name", null).set("renamed_actor", null).set("renamed_category", "comedy").build();
        Set<GenericRecord> expected = ImmutableSet.of(getBobRecord1(avroOutputSchema), getBobRecord2(avroOutputSchema), getAlexRecord1(avroOutputSchema), getAlexRecord2(avroOutputSchema), getCathieRecord1(avroOutputSchema), getAvatarRecord1(avroOutputSchema), humtumRecord, sultanRecord);
        Assert.assertEquals(expected, actual);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
    return null;
}

Example 62 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project hydrator-plugins by cdapio.

the class StructuredToAvroTransformer method transform.

@Override
public GenericRecord transform(StructuredRecord structuredRecord, io.cdap.cdap.api.data.schema.Schema schema) throws IOException {
    io.cdap.cdap.api.data.schema.Schema structuredRecordSchema = structuredRecord.getSchema();
    Schema avroSchema = getAvroSchema(schema);
    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(avroSchema);
    for (Schema.Field field : avroSchema.getFields()) {
        String fieldName = field.name();
        io.cdap.cdap.api.data.schema.Schema.Field schemaField = structuredRecordSchema.getField(fieldName);
        if (schemaField == null) {
            throw new IllegalArgumentException("Input record does not contain the " + fieldName + " field.");
        }
        recordBuilder.set(fieldName, convertField(structuredRecord.get(fieldName), schemaField));
    }
    return recordBuilder.build();
}

Example 63 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project storm by apache.

the class AvroGenericRecordBoltTest method setupClass.

@BeforeClass
public static void setupClass() {
    Schema.Parser parser = new Schema.Parser();
    schema1 = parser.parse(schemaV1);
    parser = new Schema.Parser();
    schema2 = parser.parse(schemaV2);
    GenericRecordBuilder builder1 = new GenericRecordBuilder(schema1);
    builder1.set("foo1", "bar1");
    builder1.set("int1", 1);
    tuple1 = generateTestTuple(builder1.build());
    GenericRecordBuilder builder2 = new GenericRecordBuilder(schema2);
    builder2.set("foo1", "bar2");
    builder2.set("int1", 2);
    tuple2 = generateTestTuple(builder2.build());
}

Example 64 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project drill by apache.

the class KafkaMessageGenerator method populateAvroMsgIntoKafka.

public void populateAvroMsgIntoKafka(String topic, int numMsg) {
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
    try (KafkaProducer<Object, GenericRecord> producer = new KafkaProducer<>(producerProperties)) {
        Schema.Parser parser = new Schema.Parser();
        String userSchema = "{\"type\":\"record\"," + "\"name\":\"myrecord\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}," + "{\"name\":\"key2\",\"type\":\"int\"}," + "{\"name\":\"key3\",\"type\":\"boolean\"}," + "{\"name\":\"key5\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}," + "{\"name\":\"key6\",\"type\":{\"type\":\"record\",\"name\":\"myrecord6\",\"fields\":[" + "{\"name\":\"key61\",\"type\":\"double\"}," + "{\"name\":\"key62\",\"type\":\"double\"}]}}]}";
        Schema valueSchema = parser.parse(userSchema);
        GenericRecordBuilder valueBuilder = new GenericRecordBuilder(valueSchema);
        String key1Schema = "{\"type\":\"record\"," + "\"name\":\"key1record\"," + "\"fields\":[" + "{\"name\":\"key1\",\"type\":\"string\"}]}\"";
        Schema keySchema = parser.parse(key1Schema);
        GenericRecordBuilder keyBuilder = new GenericRecordBuilder(keySchema);
        Random rand = new Random();
        for (int i = 0; i < numMsg; ++i) {
            // value record
            String key1 = UUID.randomUUID().toString();
            valueBuilder.set("key1", key1);
            valueBuilder.set("key2", rand.nextInt());
            valueBuilder.set("key3", rand.nextBoolean());
            List<Integer> list = Lists.newArrayList();
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            list.add(rand.nextInt(100));
            valueBuilder.set("key5", list);
            GenericRecordBuilder innerBuilder = new GenericRecordBuilder(valueSchema.getField("key6").schema());
            innerBuilder.set("key61", rand.nextDouble());
            innerBuilder.set("key62", rand.nextDouble());
            valueBuilder.set("key6", innerBuilder.build());
            Record producerRecord = valueBuilder.build();
            // key record
            keyBuilder.set("key1", key1);
            Record keyRecord = keyBuilder.build();
            ProducerRecord<Object, GenericRecord> record = new ProducerRecord<>(topic, keyRecord, producerRecord);
            producer.send(record);
        }
    }
}

Example 65 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtilsTest method testUnionFieldInAvroSchema.

@Test
public void testUnionFieldInAvroSchema() {
    List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
    List<org.apache.avro.Schema> unionFields = Lists.newArrayList();
    unionFields.add(org.apache.avro.Schema.create(Type.INT));
    unionFields.add(org.apache.avro.Schema.create(Type.STRING));
    fields.add(new org.apache.avro.Schema.Field("union", org.apache.avro.Schema.createUnion(unionFields), "", null));
    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields);
    OneOfType oneOfType = OneOfType.create(Field.of("int", FieldType.INT32), Field.of("string", FieldType.STRING));
    Schema expectedSchema = Schema.builder().addLogicalTypeField("union", oneOfType).build();
    assertEquals(expectedSchema, AvroUtils.toBeamSchema(avroSchema));
    GenericRecord genericRecord = new GenericRecordBuilder(avroSchema).set("union", 23423).build();
    Row expectedRow = Row.withSchema(expectedSchema).addValue(oneOfType.createValue(0, 23423)).build();
    assertEquals(expectedRow, AvroUtils.toBeamRowStrict(genericRecord, expectedSchema));
}