Examples with HoodieWriteConfig org.apache.hudi.config.HoodieWriteConfig used on opensource projects

Example 31 with HoodieWriteConfig

use of org.apache.hudi.config.HoodieWriteConfig in project hudi by apache.

the class TestHoodieSparkMergeOnReadTableClustering method testClusteringWithNoBaseFiles.

@ParameterizedTest
@ValueSource(booleans = { true, false })
void testClusteringWithNoBaseFiles(boolean doUpdates) throws Exception {
    // set low compaction small File Size to generate more file groups.
    HoodieWriteConfig.Builder cfgBuilder = HoodieWriteConfig.newBuilder().forTable("test-trip-table").withPath(basePath()).withSchema(TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2).withDeleteParallelism(2).withAutoCommit(true).withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(10L).withInlineCompaction(false).withMaxNumDeltaCommitsBeforeCompaction(1).build()).withStorageConfig(HoodieStorageConfig.newBuilder().hfileMaxFileSize(1024 * 1024 * 1024).parquetMaxFileSize(1024 * 1024 * 1024).build()).withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(new FileSystemViewStorageConfig.Builder().withEnableBackupForRemoteFileSystemView(false).build()).withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).withClusteringConfig(HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0).withInlineClustering(true).withInlineClusteringNumCommits(1).build()).withRollbackUsingMarkers(false);
    HoodieWriteConfig cfg = cfgBuilder.build();
    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, cfg.getProps());
    HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
    try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
        // test 2 inserts
        String newCommitTime = "001";
        client.startCommitWithTime(newCommitTime);
        List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 400);
        Stream<HoodieBaseFile> dataFiles = insertRecordsToMORTable(metaClient, records.subList(0, 200), client, cfg, newCommitTime);
        assertTrue(!dataFiles.findAny().isPresent(), "should not have any base files");
        newCommitTime = "002";
        client.startCommitWithTime(newCommitTime);
        dataFiles = insertRecordsToMORTable(metaClient, records.subList(200, 400), client, cfg, newCommitTime);
        assertTrue(!dataFiles.findAny().isPresent(), "should not have any base files");
        // run updates
        if (doUpdates) {
            newCommitTime = "003";
            client.startCommitWithTime(newCommitTime);
            records = dataGen.generateUpdates(newCommitTime, 100);
            updateRecordsInMORTable(metaClient, records, client, cfg, newCommitTime, false);
        }
        HoodieTable hoodieTable = HoodieSparkTable.create(cfg, context(), metaClient);
        hoodieTable.getHoodieView().sync();
        FileStatus[] allBaseFiles = listAllBaseFilesInPath(hoodieTable);
        // expect 0 base files for each partition
        assertEquals(0, allBaseFiles.length);
        String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
        metaClient = HoodieTableMetaClient.reload(metaClient);
        hoodieTable = HoodieSparkTable.create(cfg, context(), metaClient);
        // verify log files are included in clustering plan for each partition.
        assertEquals(dataGen.getPartitionPaths().length, hoodieTable.getFileSystemView().getFileGroupsInPendingClustering().map(Pair::getLeft).count());
        // do the clustering and validate
        doClusteringAndValidate(client, clusteringCommitTime, metaClient, cfg, dataGen);
    }
}

Example 32 with HoodieWriteConfig

use of org.apache.hudi.config.HoodieWriteConfig in project hudi by apache.

the class TestHoodieSparkMergeOnReadTableClustering method testClustering.

@ParameterizedTest
@MethodSource
void testClustering(boolean doUpdates, boolean populateMetaFields, boolean preserveCommitMetadata) throws Exception {
    // set low compaction small File Size to generate more file groups.
    HoodieWriteConfig.Builder cfgBuilder = HoodieWriteConfig.newBuilder().forTable("test-trip-table").withPath(basePath()).withSchema(TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2).withDeleteParallelism(2).withAutoCommit(true).withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(10L).withInlineCompaction(false).withMaxNumDeltaCommitsBeforeCompaction(1).build()).withStorageConfig(HoodieStorageConfig.newBuilder().hfileMaxFileSize(1024 * 1024 * 1024).parquetMaxFileSize(1024 * 1024 * 1024).build()).withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(new FileSystemViewStorageConfig.Builder().withEnableBackupForRemoteFileSystemView(false).build()).withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).withClusteringConfig(HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10).withClusteringTargetPartitions(0).withInlineClustering(true).withInlineClusteringNumCommits(1).withPreserveHoodieCommitMetadata(preserveCommitMetadata).build()).withRollbackUsingMarkers(false);
    addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
    HoodieWriteConfig cfg = cfgBuilder.build();
    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, cfg.getProps());
    HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
    try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
        /*
       * Write 1 (only inserts)
       */
        String newCommitTime = "001";
        client.startCommitWithTime(newCommitTime);
        List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 400);
        Stream<HoodieBaseFile> dataFiles = insertRecordsToMORTable(metaClient, records.subList(0, 200), client, cfg, newCommitTime);
        assertTrue(dataFiles.findAny().isPresent(), "should list the base files we wrote in the delta commit");
        /*
       * Write 2 (more inserts to create new files)
       */
        // we already set small file size to small number to force inserts to go into new file.
        newCommitTime = "002";
        client.startCommitWithTime(newCommitTime);
        dataFiles = insertRecordsToMORTable(metaClient, records.subList(200, 400), client, cfg, newCommitTime);
        assertTrue(dataFiles.findAny().isPresent(), "should list the base files we wrote in the delta commit");
        if (doUpdates) {
            /*
         * Write 3 (updates)
         */
            newCommitTime = "003";
            client.startCommitWithTime(newCommitTime);
            records = dataGen.generateUpdates(newCommitTime, 100);
            updateRecordsInMORTable(metaClient, records, client, cfg, newCommitTime, false);
        }
        HoodieTable hoodieTable = HoodieSparkTable.create(cfg, context(), metaClient);
        hoodieTable.getHoodieView().sync();
        FileStatus[] allFiles = listAllBaseFilesInPath(hoodieTable);
        // expect 2 base files for each partition
        assertEquals(dataGen.getPartitionPaths().length * 2, allFiles.length);
        String clusteringCommitTime = client.scheduleClustering(Option.empty()).get().toString();
        metaClient = HoodieTableMetaClient.reload(metaClient);
        hoodieTable = HoodieSparkTable.create(cfg, context(), metaClient);
        // verify all files are included in clustering plan.
        assertEquals(allFiles.length, hoodieTable.getFileSystemView().getFileGroupsInPendingClustering().map(Pair::getLeft).count());
        // Do the clustering and validate
        doClusteringAndValidate(client, clusteringCommitTime, metaClient, cfg, dataGen);
    }
}

Example 33 with HoodieWriteConfig

use of org.apache.hudi.config.HoodieWriteConfig in project hudi by apache.

the class TestHoodieSparkMergeOnReadTableCompaction method testWriteDuringCompaction.

@Test
public void testWriteDuringCompaction() throws IOException {
    HoodieWriteConfig config = HoodieWriteConfig.newBuilder().forTable("test-trip-table").withPath(basePath()).withSchema(TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2).withAutoCommit(false).withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(1).build()).withStorageConfig(HoodieStorageConfig.newBuilder().parquetMaxFileSize(1024).build()).withLayoutConfig(HoodieLayoutConfig.newBuilder().withLayoutType(HoodieStorageLayout.LayoutType.BUCKET.name()).withLayoutPartitioner(SparkBucketIndexPartitioner.class.getName()).build()).withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build()).build();
    metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, config.getProps());
    client = getHoodieWriteClient(config);
    // write data and commit
    writeData(HoodieActiveTimeline.createNewInstantTime(), 100, true);
    // write data again, and in the case of bucket index, all records will go into log files (we use a small max_file_size)
    writeData(HoodieActiveTimeline.createNewInstantTime(), 100, true);
    Assertions.assertEquals(200, readTableTotalRecordsNum());
    // schedule compaction
    String compactionTime = (String) client.scheduleCompaction(Option.empty()).get();
    // write data, and do not commit. those records should not visible to reader
    String insertTime = HoodieActiveTimeline.createNewInstantTime();
    List<WriteStatus> writeStatuses = writeData(insertTime, 100, false);
    Assertions.assertEquals(200, readTableTotalRecordsNum());
    // commit the write. The records should be visible now even though the compaction does not complete.
    client.commitStats(insertTime, writeStatuses.stream().map(WriteStatus::getStat).collect(Collectors.toList()), Option.empty(), metaClient.getCommitActionType());
    Assertions.assertEquals(300, readTableTotalRecordsNum());
    // after the compaction, total records should remain the same
    config.setValue(AUTO_COMMIT_ENABLE, "true");
    client.compact(compactionTime);
    Assertions.assertEquals(300, readTableTotalRecordsNum());
}

Example 34 with HoodieWriteConfig

use of org.apache.hudi.config.HoodieWriteConfig in project hudi by apache.

the class TestHoodieSparkMergeOnReadTableIncrementalRead method testIncrementalReadsWithCompaction.

// test incremental read does not go past compaction instant for RO views
// For RT views, incremental read can go past compaction
@Test
public void testIncrementalReadsWithCompaction() throws Exception {
    // use only one partition for this test
    final String partitionPath = "2020/02/20";
    final HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(new String[] { partitionPath });
    Properties props = new Properties();
    props.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.PARQUET.toString());
    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, props);
    HoodieWriteConfig cfg = getConfigBuilder(true).build();
    try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
        /*
       * Write 1 (only inserts)
       */
        String commitTime1 = "001";
        client.startCommitWithTime(commitTime1);
        List<HoodieRecord> records001 = dataGen.generateInserts(commitTime1, 200);
        Stream<HoodieBaseFile> dataFiles = insertRecordsToMORTable(metaClient, records001, client, cfg, commitTime1);
        assertTrue(dataFiles.findAny().isPresent(), "should list the base files we wrote in the delta commit");
        // verify only one base file shows up with commit time 001
        FileStatus[] snapshotROFiles = getROSnapshotFiles(partitionPath);
        validateFiles(partitionPath, 1, snapshotROFiles, false, roSnapshotJobConf, 200, commitTime1);
        FileStatus[] incrementalROFiles = getROIncrementalFiles(partitionPath, true);
        validateFiles(partitionPath, 1, incrementalROFiles, false, roJobConf, 200, commitTime1);
        Path firstFilePath = incrementalROFiles[0].getPath();
        FileStatus[] incrementalRTFiles = getRTIncrementalFiles(partitionPath);
        validateFiles(partitionPath, 1, incrementalRTFiles, true, rtJobConf, 200, commitTime1);
        assertEquals(firstFilePath, incrementalRTFiles[0].getPath());
        /*
       * Write 2 (updates)
       */
        String updateTime = "004";
        client.startCommitWithTime(updateTime);
        List<HoodieRecord> records004 = dataGen.generateUpdates(updateTime, 100);
        updateRecordsInMORTable(metaClient, records004, client, cfg, updateTime, false);
        // verify RO incremental reads - only one base file shows up because updates to into log files
        incrementalROFiles = getROIncrementalFiles(partitionPath, false);
        validateFiles(partitionPath, 1, incrementalROFiles, false, roJobConf, 200, commitTime1);
        assertEquals(firstFilePath, incrementalROFiles[0].getPath());
        // verify RT incremental reads includes updates also
        incrementalRTFiles = getRTIncrementalFiles(partitionPath);
        validateFiles(partitionPath, 1, incrementalRTFiles, true, rtJobConf, 200, commitTime1, updateTime);
        // request compaction, but do not perform compaction
        String compactionCommitTime = "005";
        client.scheduleCompactionAtInstant("005", Option.empty());
        // verify RO incremental reads - only one base file shows up because updates go into log files
        incrementalROFiles = getROIncrementalFiles(partitionPath, true);
        validateFiles(partitionPath, 1, incrementalROFiles, false, roJobConf, 200, commitTime1);
        // verify RT incremental reads includes updates also
        incrementalRTFiles = getRTIncrementalFiles(partitionPath);
        validateFiles(partitionPath, 1, incrementalRTFiles, true, rtJobConf, 200, commitTime1, updateTime);
        // write 3 - more inserts
        String insertsTime = "006";
        List<HoodieRecord> records006 = dataGen.generateInserts(insertsTime, 200);
        client.startCommitWithTime(insertsTime);
        dataFiles = insertRecordsToMORTable(metaClient, records006, client, cfg, insertsTime);
        assertTrue(dataFiles.findAny().isPresent(), "should list the base files we wrote in the delta commit");
        // verify new write shows up in snapshot mode even though there is pending compaction
        snapshotROFiles = getROSnapshotFiles(partitionPath);
        validateFiles(partitionPath, 2, snapshotROFiles, false, roSnapshotJobConf, 400, commitTime1, insertsTime);
        incrementalROFiles = getROIncrementalFiles(partitionPath, true);
        assertEquals(firstFilePath, incrementalROFiles[0].getPath());
        // verify 006 does not show up in RO mode because of pending compaction
        validateFiles(partitionPath, 1, incrementalROFiles, false, roJobConf, 200, commitTime1);
        // verify that if stopAtCompaction is disabled, inserts from "insertsTime" show up
        incrementalROFiles = getROIncrementalFiles(partitionPath, false);
        validateFiles(partitionPath, 2, incrementalROFiles, false, roJobConf, 400, commitTime1, insertsTime);
        // verify 006 shows up in RT views
        incrementalRTFiles = getRTIncrementalFiles(partitionPath);
        validateFiles(partitionPath, 2, incrementalRTFiles, true, rtJobConf, 400, commitTime1, updateTime, insertsTime);
        // perform the scheduled compaction
        client.compact(compactionCommitTime);
        // verify new write shows up in snapshot mode after compaction is complete
        snapshotROFiles = getROSnapshotFiles(partitionPath);
        validateFiles(partitionPath, 2, snapshotROFiles, false, roSnapshotJobConf, 400, commitTime1, compactionCommitTime, insertsTime);
        incrementalROFiles = getROIncrementalFiles(partitionPath, "002", -1, true);
        assertTrue(incrementalROFiles.length == 2);
        // verify 006 shows up because of pending compaction
        validateFiles(partitionPath, 2, incrementalROFiles, false, roJobConf, 400, commitTime1, compactionCommitTime, insertsTime);
    }
}

Example 35 with HoodieWriteConfig

use of org.apache.hudi.config.HoodieWriteConfig in project hudi by apache.

the class TestHoodieSparkMergeOnReadTableInsertUpdateDelete method testSimpleInsertsGeneratedIntoLogFiles.

@Test
public void testSimpleInsertsGeneratedIntoLogFiles() throws Exception {
    // insert 100 records
    // Setting IndexType to be InMemory to simulate Global Index nature
    HoodieWriteConfig config = getConfigBuilder(false, HoodieIndex.IndexType.INMEMORY).build();
    Properties properties = new Properties();
    properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().toString());
    HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, properties);
    try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
        String newCommitTime = "100";
        writeClient.startCommitWithTime(newCommitTime);
        HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
        List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
        JavaRDD<HoodieRecord> recordsRDD = jsc().parallelize(records, 1);
        JavaRDD<WriteStatus> statuses = writeClient.insert(recordsRDD, newCommitTime);
        writeClient.commit(newCommitTime, statuses);
        HoodieTable table = HoodieSparkTable.create(config, context(), metaClient);
        table.getHoodieView().sync();
        TableFileSystemView.SliceView tableRTFileSystemView = table.getSliceView();
        long numLogFiles = 0;
        for (String partitionPath : dataGen.getPartitionPaths()) {
            List<FileSlice> allSlices = tableRTFileSystemView.getLatestFileSlices(partitionPath).collect(Collectors.toList());
            assertEquals(0, allSlices.stream().filter(fileSlice -> fileSlice.getBaseFile().isPresent()).count());
            assertTrue(allSlices.stream().anyMatch(fileSlice -> fileSlice.getLogFiles().count() > 0));
            long logFileCount = allSlices.stream().filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();
            if (logFileCount > 0) {
                // check the log versions start from the base version
                assertTrue(allSlices.stream().map(slice -> slice.getLogFiles().findFirst().get().getLogVersion()).allMatch(version -> version.equals(HoodieLogFile.LOGFILE_BASE_VERSION)));
            }
            numLogFiles += logFileCount;
        }
        assertTrue(numLogFiles > 0);
        // Do a compaction
        String instantTime = writeClient.scheduleCompaction(Option.empty()).get().toString();
        HoodieWriteMetadata<JavaRDD<WriteStatus>> compactionMetadata = writeClient.compact(instantTime);
        String extension = table.getBaseFileExtension();
        Collection<List<HoodieWriteStat>> stats = compactionMetadata.getCommitMetadata().get().getPartitionToWriteStats().values();
        assertEquals(numLogFiles, stats.stream().flatMap(Collection::stream).filter(state -> state.getPath().contains(extension)).count());
        assertEquals(numLogFiles, stats.stream().mapToLong(Collection::size).sum());
        writeClient.commitCompaction(instantTime, compactionMetadata.getCommitMetadata().get(), Option.empty());
    }
}