Examples with GenericRecordBuilder org.apache.avro.generic.GenericRecordBuilder used on opensource projects

Example 71 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class ConfluentSchemaRegistryDeserializerProviderTest method testDeserialize.

@Test
public void testDeserialize() {
    // Test deserializing evolved schema.
    // Verify that records from older schemas are deserialized to the latest schema
    String schemaRegistryUrl = "mock://my-scope-name";
    String subject = "mytopic";
    SchemaRegistryClient mockRegistryClient = mockSchemaRegistryClient(schemaRegistryUrl, subject);
    Map<String, Object> map = new HashMap<>();
    map.put(AbstractKafkaAvroSerDeConfig.AUTO_REGISTER_SCHEMAS, true);
    map.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, schemaRegistryUrl);
    Serializer<GenericRecord> serializer = (Serializer) new KafkaAvroSerializer(mockRegistryClient);
    serializer.configure(map, true);
    byte[] bytes = serializer.serialize(subject, new GenericRecordBuilder(AVRO_SCHEMA_V1).set("name", "KeyName").set("age", 1).set("favorite_number", 2).set("favorite_color", "color3").build());
    Object deserialized = mockDeserializerProvider(schemaRegistryUrl, subject, null).getDeserializer(new HashMap<>(), true).deserialize(subject, bytes);
    GenericRecord expected = new GenericRecordBuilder(AVRO_SCHEMA).set("name", "KeyName").set("favorite_number", 2).set("favorite_color", "color3").build();
    assertEquals(expected, deserialized);
}

Example 72 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtils method toGenericRecord.

/**
 * Convert from a Beam Row to an AVRO GenericRecord. If a Schema is not provided, one is inferred
 * from the Beam schema on the row.
 */
public static GenericRecord toGenericRecord(Row row, org.apache.avro.@Nullable Schema avroSchema) {
    Schema beamSchema = row.getSchema();
    // schema.
    if (avroSchema != null && avroSchema.getFields().size() != beamSchema.getFieldCount()) {
        throw new IllegalArgumentException("AVRO schema doesn't match row schema. Row schema " + beamSchema + ". AVRO schema + " + avroSchema);
    }
    if (avroSchema == null) {
        avroSchema = toAvroSchema(beamSchema);
    }
    GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
    for (int i = 0; i < beamSchema.getFieldCount(); ++i) {
        Schema.Field field = beamSchema.getField(i);
        builder.set(field.getName(), genericFromBeamField(field.getType(), avroSchema.getField(field.getName()).schema(), row.getValue(i)));
    }
    return builder.build();
}

Example 73 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class ParquetIOTest method generateRequestedRecords.

private List<GenericRecord> generateRequestedRecords(long count) {
    ArrayList<GenericRecord> data = new ArrayList<>();
    GenericRecordBuilder builder = new GenericRecordBuilder(REQUESTED_ENCODER_SCHEMA);
    for (int i = 0; i < count; i++) {
        GenericRecord record = builder.set("id", Integer.toString(i)).set("name", null).build();
        data.add(record);
    }
    return data;
}

Example 74 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class ParquetIOTest method generateGenericRecords.

private List<GenericRecord> generateGenericRecords(long count) {
    List<GenericRecord> data = new ArrayList<>();
    GenericRecordBuilder builder = new GenericRecordBuilder(SCHEMA);
    for (int i = 0; i < count; i++) {
        int index = i % SCIENTISTS.length;
        GenericRecord record = builder.set("name", SCIENTISTS[index]).set("id", Integer.toString(i)).build();
        data.add(record);
    }
    return data;
}

Example 75 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project hazelcast by hazelcast.

the class SqlAvroTest method test_topLevelFieldExtraction.

@Test
public void test_topLevelFieldExtraction() {
    String name = createRandomTopic();
    sqlService.execute("CREATE MAPPING " + name + " (" + "id INT EXTERNAL NAME \"__key.id\"" + ", name VARCHAR" + ") TYPE " + KafkaSqlConnector.TYPE_NAME + ' ' + "OPTIONS ( " + '\'' + OPTION_KEY_FORMAT + "'='" + AVRO_FORMAT + '\'' + ", '" + OPTION_VALUE_FORMAT + "'='" + AVRO_FORMAT + '\'' + ", 'bootstrap.servers'='" + kafkaTestSupport.getBrokerConnectionString() + '\'' + ", 'schema.registry.url'='" + schemaRegistry.getURI() + '\'' + ", 'auto.offset.reset'='earliest'" + ")");
    sqlService.execute("INSERT INTO " + name + " VALUES (1, 'Alice')");
    assertRowsEventuallyInAnyOrder("SELECT __key, this FROM " + name, singletonList(new Row(new GenericRecordBuilder(intSchema("id")).set("id", 1).build(), new GenericRecordBuilder(stringSchema("name")).set("name", "Alice").build())));
}