Examples with AvroFileDeltaInputWriter org.apache.hudi.integ.testsuite.writer.AvroFileDeltaInputWriter used on opensource projects

Example 1 with AvroFileDeltaInputWriter

use of org.apache.hudi.integ.testsuite.writer.AvroFileDeltaInputWriter in project hudi by apache.

the class TestFileDeltaInputWriter method testAvroFileSinkCreateNewWriter.

@Test
public void testAvroFileSinkCreateNewWriter() throws IOException {
    // 1. Create a Avro File Sink Writer
    DeltaInputWriter<GenericRecord> fileSinkWriter = new AvroFileDeltaInputWriter(jsc.hadoopConfiguration(), dfsBasePath, schemaProvider.getSourceSchema().toString(), 1024 * 1024L);
    GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schemaProvider.getSourceSchema());
    // 2. Generate 100 avro payloads and write them to an avro file
    IntStream.range(0, 100).forEach(a -> {
        try {
            fileSinkWriter.writeData(payloadGenerator.getNewPayload());
        } catch (IOException io) {
            throw new UncheckedIOException(io);
        }
    });
    fileSinkWriter.close();
    String oldFilePath = fileSinkWriter.getDeltaWriteStats().getFilePath();
    assertFalse(oldFilePath == null);
    DeltaInputWriter<GenericRecord> newFileSinkWriter = fileSinkWriter.getNewWriter();
    newFileSinkWriter.close();
    DeltaWriteStats newStats = newFileSinkWriter.getDeltaWriteStats();
    assertEquals(newStats.getBytesWritten(), 3674);
    assertEquals(newStats.getRecordsWritten(), 0);
    assertTrue(newStats.getFilePath() != null);
}

Example 2 with AvroFileDeltaInputWriter

use of org.apache.hudi.integ.testsuite.writer.AvroFileDeltaInputWriter in project hudi by apache.

the class TestFileDeltaInputWriter method testAvroFileSinkWriter.

@Test
public void testAvroFileSinkWriter() throws IOException {
    // 1. Create a Avro File Sink Writer
    DeltaInputWriter<GenericRecord> fileSinkWriter = new AvroFileDeltaInputWriter(jsc.hadoopConfiguration(), dfsBasePath + "/input", schemaProvider.getSourceSchema().toString(), 1024 * 1024L);
    GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schemaProvider.getSourceSchema());
    // 2. Generate 100 avro payloads and write them to an avro file
    IntStream.range(0, 100).forEach(a -> {
        try {
            fileSinkWriter.writeData(payloadGenerator.getNewPayload());
        } catch (IOException io) {
            throw new UncheckedIOException(io);
        }
    });
    fileSinkWriter.close();
    DeltaWriteStats deltaWriteStats = fileSinkWriter.getDeltaWriteStats();
    FileSystem fs = FSUtils.getFs(dfsBasePath, jsc.hadoopConfiguration());
    FileStatus[] fileStatuses = fs.listStatus(new Path(deltaWriteStats.getFilePath()));
    // Atleast 1 file was written
    assertEquals(1, fileStatuses.length);
    // File length should be greater than 0
    assertTrue(fileStatuses[0].getLen() > 0);
    // File length should be the same as the number of bytes written
    assertTrue(deltaWriteStats.getBytesWritten() > 0);
    List<String> paths = Arrays.asList(fs.globStatus(new Path(dfsBasePath + "/*/*.avro"))).stream().map(f -> f.getPath().toString()).collect(Collectors.toList());
    JavaRDD<GenericRecord> writtenRecords = SparkBasedReader.readAvro(sparkSession, schemaProvider.getSourceSchema().toString(), paths, Option.empty(), Option.empty());
    // Number of records written should be 100
    assertEquals(writtenRecords.count(), 100);
    // Number of records in file should match with the stats
    assertEquals(writtenRecords.count(), deltaWriteStats.getRecordsWritten());
}