Examples with RDDCustomColumnsSortPartitioner org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner used on opensource projects

Example 1 with RDDCustomColumnsSortPartitioner

use of org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner in project hudi by apache.

the class TestHoodieClientOnCopyOnWriteStorage method testBulkInsertWithCustomPartitioner.

@Test
public void testBulkInsertWithCustomPartitioner() {
    HoodieWriteConfig config = getConfigBuilder().withRollbackUsingMarkers(true).build();
    try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
        final String commitTime1 = "001";
        client.startCommitWithTime(commitTime1);
        List<HoodieRecord> inserts1 = dataGen.generateInserts(commitTime1, 100);
        JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 10);
        BulkInsertPartitioner<JavaRDD<HoodieRecord>> partitioner = new RDDCustomColumnsSortPartitioner(new String[] { "rider" }, HoodieTestDataGenerator.AVRO_SCHEMA, false);
        List<WriteStatus> statuses = client.bulkInsert(insertRecordsRDD1, commitTime1, Option.of(partitioner)).collect();
        assertNoWriteErrors(statuses);
    }
}