Examples with SchemaProvider org.apache.hudi.schema.SchemaProvider used on opensource projects

Example 1 with SchemaProvider

use of org.apache.hudi.schema.SchemaProvider in project hudi by apache.

the class TestAbstractConnectWriter method testAbstractWriterForAllFormats.

@ParameterizedTest
@EnumSource(value = TestInputFormats.class)
public void testAbstractWriterForAllFormats(TestInputFormats inputFormats) throws Exception {
    Schema schema = schemaProvider.getSourceSchema();
    List<?> inputRecords;
    List<HoodieRecord> expectedRecords;
    String formatConverter;
    switch(inputFormats) {
        case JSON_STRING:
            formatConverter = AbstractConnectWriter.KAFKA_STRING_CONVERTER;
            GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(schema, schema);
            inputRecords = SchemaTestUtil.generateTestJsonRecords(0, NUM_RECORDS);
            expectedRecords = ((List<String>) inputRecords).stream().map(s -> {
                try {
                    return HoodieAvroUtils.rewriteRecord((GenericRecord) reader.read(null, DecoderFactory.get().jsonDecoder(schema, s)), schema);
                } catch (IOException exception) {
                    throw new HoodieException("Error converting JSON records to AVRO");
                }
            }).map(p -> convertToHoodieRecords(p, p.get(RECORD_KEY_INDEX).toString(), "000/00/00")).collect(Collectors.toList());
            break;
        case AVRO:
            formatConverter = AbstractConnectWriter.KAFKA_AVRO_CONVERTER;
            inputRecords = SchemaTestUtil.generateTestRecords(0, NUM_RECORDS);
            expectedRecords = inputRecords.stream().map(s -> HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema)).map(p -> convertToHoodieRecords(p, p.get(RECORD_KEY_INDEX).toString(), "000/00/00")).collect(Collectors.toList());
            break;
        default:
            throw new HoodieException("Unknown test scenario " + inputFormats);
    }
    configs = KafkaConnectConfigs.newBuilder().withProperties(Collections.singletonMap(KafkaConnectConfigs.KAFKA_VALUE_CONVERTER, formatConverter)).build();
    AbstractHudiConnectWriterTestWrapper writer = new AbstractHudiConnectWriterTestWrapper(configs, keyGenerator, schemaProvider);
    for (int i = 0; i < NUM_RECORDS; i++) {
        writer.writeRecord(getNextKafkaRecord(inputRecords.get(i)));
    }
    validateRecords(writer.getWrittenRecords(), expectedRecords);
}