Examples with GenericRecordBuilder org.apache.avro.generic.GenericRecordBuilder used on opensource projects

Example 66 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtilsTest method testJdbcLogicalVarCharRowDataToGenericRecord.

@Test
public void testJdbcLogicalVarCharRowDataToGenericRecord() {
    Schema beamSchema = Schema.builder().addField(Field.of("my_varchar_field", FieldType.logicalType(JdbcType.StringType.varchar(10)))).addField(Field.of("my_longvarchar_field", FieldType.logicalType(JdbcType.StringType.longvarchar(50)))).addField(Field.of("my_nvarchar_field", FieldType.logicalType(JdbcType.StringType.nvarchar(10)))).addField(Field.of("my_longnvarchar_field", FieldType.logicalType(JdbcType.StringType.longnvarchar(50)))).build();
    Row rowData = Row.withSchema(beamSchema).addValue("varchar_value").addValue("longvarchar_value").addValue("nvarchar_value").addValue("longnvarchar_value").build();
    org.apache.avro.Schema avroSchema = AvroUtils.toAvroSchema(beamSchema);
    GenericRecord expectedRecord = new GenericRecordBuilder(avroSchema).set("my_varchar_field", "varchar_value").set("my_longvarchar_field", "longvarchar_value").set("my_nvarchar_field", "nvarchar_value").set("my_longnvarchar_field", "longnvarchar_value").build();
    assertEquals(expectedRecord, AvroUtils.toGenericRecord(rowData, avroSchema));
}

Example 67 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtilsTest method testNullableFieldsInBeamSchema.

@Test
public void testNullableFieldsInBeamSchema() {
    Schema beamSchema = Schema.builder().addNullableField("int", FieldType.INT32).addArrayField("array", FieldType.INT32.withNullable(true)).addMapField("map", FieldType.STRING, FieldType.INT32.withNullable(true)).build();
    List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
    fields.add(new org.apache.avro.Schema.Field("int", ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT)), "", null));
    fields.add(new org.apache.avro.Schema.Field("array", org.apache.avro.Schema.createArray(ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))), "", null));
    fields.add(new org.apache.avro.Schema.Field("map", org.apache.avro.Schema.createMap(ReflectData.makeNullable(org.apache.avro.Schema.create(Type.INT))), "", null));
    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields);
    assertEquals(avroSchema, AvroUtils.toAvroSchema(beamSchema));
    Map<Utf8, Object> nullMapUtf8 = Maps.newHashMap();
    nullMapUtf8.put(new Utf8("k1"), null);
    Map<String, Object> nullMapString = Maps.newHashMap();
    nullMapString.put("k1", null);
    GenericRecord expectedGenericRecord = new GenericRecordBuilder(avroSchema).set("int", null).set("array", Lists.newArrayList((Object) null)).set("map", nullMapUtf8).build();
    Row row = Row.withSchema(beamSchema).addValue(null).addValue(Lists.newArrayList((Object) null)).addValue(nullMapString).build();
    assertEquals(expectedGenericRecord, AvroUtils.toGenericRecord(row, avroSchema));
}

Example 68 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtilsTest method testUnionFieldInBeamSchema.

@Test
public void testUnionFieldInBeamSchema() {
    OneOfType oneOfType = OneOfType.create(Field.of("int", FieldType.INT32), Field.of("string", FieldType.STRING));
    Schema beamSchema = Schema.builder().addLogicalTypeField("union", oneOfType).build();
    List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
    List<org.apache.avro.Schema> unionFields = Lists.newArrayList();
    unionFields.add(org.apache.avro.Schema.create(Type.INT));
    unionFields.add(org.apache.avro.Schema.create(Type.STRING));
    fields.add(new org.apache.avro.Schema.Field("union", org.apache.avro.Schema.createUnion(unionFields), "", null));
    org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("topLevelRecord", null, null, false, fields);
    GenericRecord expectedGenericRecord = new GenericRecordBuilder(avroSchema).set("union", 23423).build();
    Row row = Row.withSchema(beamSchema).addValue(oneOfType.createValue(0, 23423)).build();
    assertEquals(expectedGenericRecord, AvroUtils.toGenericRecord(row, avroSchema));
}

Example 69 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class AvroUtilsTest method getGenericRecord.

private static GenericRecord getGenericRecord() {
    LogicalType decimalType = LogicalTypes.decimal(Integer.MAX_VALUE).addToSchema(org.apache.avro.Schema.create(Type.BYTES)).getLogicalType();
    ByteBuffer encodedDecimal = new Conversions.DecimalConversion().toBytes(BIG_DECIMAL, null, decimalType);
    return new GenericRecordBuilder(getAvroSchema()).set("bool", true).set("int", 43).set("long", 44L).set("float", (float) 44.1).set("double", (double) 44.2).set("string", new Utf8("string")).set("bytes", ByteBuffer.wrap(BYTE_ARRAY)).set("decimal", encodedDecimal).set("timestampMillis", DATE_TIME.getMillis()).set("row", getSubGenericRecord("row")).set("array", ImmutableList.of(getSubGenericRecord("array"), getSubGenericRecord("array"))).set("map", ImmutableMap.of(new Utf8("k1"), getSubGenericRecord("map"), new Utf8("k2"), getSubGenericRecord("map"))).build();
}

Example 70 with GenericRecordBuilder

use of org.apache.avro.generic.GenericRecordBuilder in project beam by apache.

the class FakeJobService method writeRowsHelper.

private void writeRowsHelper(List<TableRow> rows, Schema avroSchema, String destinationPattern, int shard) {
    String filename = destinationPattern.replace("*", String.format("%012d", shard));
    try (WritableByteChannel channel = FileSystems.create(FileSystems.matchNewResource(filename, false), MimeTypes.BINARY);
        DataFileWriter<GenericRecord> tableRowWriter = new DataFileWriter<>(new GenericDatumWriter<GenericRecord>(avroSchema)).create(avroSchema, Channels.newOutputStream(channel))) {
        for (Map<String, Object> record : rows) {
            GenericRecordBuilder genericRecordBuilder = new GenericRecordBuilder(avroSchema);
            for (Map.Entry<String, Object> field : record.entrySet()) {
                genericRecordBuilder.set(field.getKey(), field.getValue());
            }
            tableRowWriter.append(genericRecordBuilder.build());
        }
    } catch (IOException e) {
        throw new IllegalStateException(String.format("Could not create destination for extract job %s", filename), e);
    }
}