Examples with DeltaWriterAdapter org.apache.hudi.integ.testsuite.writer.DeltaWriterAdapter used on opensource projects

Example 1 with DeltaWriterAdapter

use of org.apache.hudi.integ.testsuite.writer.DeltaWriterAdapter in project hudi by apache.

the class DeltaGenerator method writeRecords.

public JavaRDD<DeltaWriteStats> writeRecords(JavaRDD<GenericRecord> records) {
    if (deltaOutputConfig.shouldDeleteOldInputData() && batchId > 1) {
        Path oldInputDir = new Path(deltaOutputConfig.getDeltaBasePath(), Integer.toString(batchId - 1));
        try {
            FileSystem fs = FSUtils.getFs(oldInputDir.toString(), deltaOutputConfig.getConfiguration());
            fs.delete(oldInputDir, true);
        } catch (IOException e) {
            log.error("Failed to delete older input data direcory " + oldInputDir, e);
        }
    }
    // The following creates a new anonymous function for iterator and hence results in serialization issues
    JavaRDD<DeltaWriteStats> ws = records.mapPartitions(itr -> {
        try {
            DeltaWriterAdapter<GenericRecord> deltaWriterAdapter = DeltaWriterFactory.getDeltaWriterAdapter(deltaOutputConfig, batchId);
            return Collections.singletonList(deltaWriterAdapter.write(itr)).iterator();
        } catch (IOException io) {
            throw new UncheckedIOException(io);
        }
    }).flatMap(List::iterator);
    batchId++;
    return ws;
}