Examples with HoodieTableType org.apache.hudi.common.model.HoodieTableType used on opensource projects

Example 1 with HoodieTableType

use of org.apache.hudi.common.model.HoodieTableType in project hudi by apache.

the class TestSparkHoodieHBaseIndex method testSimpleTagLocationAndUpdate.

@ParameterizedTest
@EnumSource(HoodieTableType.class)
public void testSimpleTagLocationAndUpdate(HoodieTableType tableType) throws Exception {
    metaClient = HoodieTestUtils.init(hadoopConf, basePath, tableType);
    final String newCommitTime = "001";
    final int numRecords = 10;
    List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, numRecords);
    JavaRDD<HoodieRecord> writeRecords = jsc().parallelize(records, 1);
    // Load to memory
    HoodieWriteConfig config = getConfig();
    SparkHoodieHBaseIndex index = new SparkHoodieHBaseIndex(config);
    try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config)) {
        metaClient = HoodieTableMetaClient.reload(metaClient);
        HoodieTable hoodieTable = HoodieSparkTable.create(config, context, metaClient);
        // Test tagLocation without any entries in index
        JavaRDD<HoodieRecord> records1 = tagLocation(index, writeRecords, hoodieTable);
        assertEquals(0, records1.filter(record -> record.isCurrentLocationKnown()).count());
        // Insert 200 records
        writeClient.startCommitWithTime(newCommitTime);
        JavaRDD<WriteStatus> writeStatues = writeClient.upsert(writeRecords, newCommitTime);
        assertNoWriteErrors(writeStatues.collect());
        // Now tagLocation for these records, hbaseIndex should not tag them since commit never occurred
        JavaRDD<HoodieRecord> records2 = tagLocation(index, writeRecords, hoodieTable);
        assertEquals(0, records2.filter(record -> record.isCurrentLocationKnown()).count());
        // Now commit this & update location of records inserted and validate no errors
        writeClient.commit(newCommitTime, writeStatues);
        // Now tagLocation for these records, hbaseIndex should tag them correctly
        metaClient = HoodieTableMetaClient.reload(metaClient);
        hoodieTable = HoodieSparkTable.create(config, context, metaClient);
        List<HoodieRecord> records3 = tagLocation(index, writeRecords, hoodieTable).collect();
        assertEquals(numRecords, records3.stream().filter(record -> record.isCurrentLocationKnown()).count());
        assertEquals(numRecords, records3.stream().map(record -> record.getKey().getRecordKey()).distinct().count());
        assertEquals(numRecords, records3.stream().filter(record -> (record.getCurrentLocation() != null && record.getCurrentLocation().getInstantTime().equals(newCommitTime))).distinct().count());
    }
}

Example 2 with HoodieTableType

use of org.apache.hudi.common.model.HoodieTableType in project hudi by apache.

the class HoodieTableSource method getStreamInputFormat.

private InputFormat<RowData, ?> getStreamInputFormat() {
    // if table does not exist, use schema from the DDL
    Schema tableAvroSchema = this.metaClient == null ? inferSchemaFromDdl() : getTableAvroSchema();
    final DataType rowDataType = AvroSchemaConverter.convertToDataType(tableAvroSchema);
    final RowType rowType = (RowType) rowDataType.getLogicalType();
    final RowType requiredRowType = (RowType) getProducedDataType().notNull().getLogicalType();
    final String queryType = this.conf.getString(FlinkOptions.QUERY_TYPE);
    if (FlinkOptions.QUERY_TYPE_SNAPSHOT.equals(queryType)) {
        final HoodieTableType tableType = HoodieTableType.valueOf(this.conf.getString(FlinkOptions.TABLE_TYPE));
        boolean emitDelete = tableType == HoodieTableType.MERGE_ON_READ;
        return mergeOnReadInputFormat(rowType, requiredRowType, tableAvroSchema, rowDataType, Collections.emptyList(), emitDelete);
    }
    String errMsg = String.format("Invalid query type : '%s', options ['%s'] are supported now", queryType, FlinkOptions.QUERY_TYPE_SNAPSHOT);
    throw new HoodieException(errMsg);
}

Example 3 with HoodieTableType

use of org.apache.hudi.common.model.HoodieTableType in project hudi by apache.

the class TestHoodieBackedMetadata method testMetadataTableWithPendingCompaction.

/**
 * Tests that virtual key configs are honored in base files after compaction in metadata table.
 */
@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testMetadataTableWithPendingCompaction(boolean simulateFailedCompaction) throws Exception {
    HoodieTableType tableType = COPY_ON_WRITE;
    init(tableType, false);
    writeConfig = getWriteConfigBuilder(true, true, false).withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).enableFullScan(true).enableMetrics(false).withMaxNumDeltaCommitsBeforeCompaction(3).build()).build();
    initWriteConfigAndMetatableWriter(writeConfig, true);
    doWriteOperation(testTable, "0000001", INSERT);
    // create an inflight compaction in metadata table.
    // not easy to create an inflight in metadata table directly, hence letting compaction succeed and then deleting the completed instant.
    // this new write is expected to trigger metadata table compaction
    String commitInstant = "0000002";
    doWriteOperation(testTable, commitInstant, INSERT);
    doWriteOperation(testTable, "0000003", INSERT);
    HoodieTableMetadata tableMetadata = metadata(writeConfig, context);
    String metadataCompactionInstant = commitInstant + "001";
    assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
    assertEquals(tableMetadata.getLatestCompactionTime().get(), metadataCompactionInstant);
    validateMetadata(testTable);
    // Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
    // for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
    java.nio.file.Path parentPath = Paths.get(metadataTableBasePath, HoodieTableMetaClient.METAFOLDER_NAME);
    java.nio.file.Path metaFilePath = parentPath.resolve(metadataCompactionInstant + HoodieTimeline.COMMIT_EXTENSION);
    java.nio.file.Path tempFilePath = FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant);
    metaClient.reloadActiveTimeline();
    testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter);
    // this validation will exercise the code path where a compaction is inflight in metadata table, but still metadata based file listing should match non
    // metadata based file listing.
    validateMetadata(testTable);
    if (simulateFailedCompaction) {
        // this should retry the compaction in metadata table.
        doWriteOperation(testTable, "0000004", INSERT);
    } else {
        // let the compaction succeed in metadata and validation should succeed.
        FileCreateUtils.renameTempToMetaFile(tempFilePath, metaFilePath);
    }
    validateMetadata(testTable);
    // add few more write and validate
    doWriteOperation(testTable, "0000005", INSERT);
    doWriteOperation(testTable, "0000006", UPSERT);
    validateMetadata(testTable);
    if (simulateFailedCompaction) {
        // trigger another compaction failure.
        metadataCompactionInstant = "0000005001";
        tableMetadata = metadata(writeConfig, context);
        assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
        assertEquals(tableMetadata.getLatestCompactionTime().get(), metadataCompactionInstant);
        // Fetch compaction Commit file and rename to some other file. completed compaction meta file should have some serialized info that table interprets
        // for future upserts. so, renaming the file here to some temp name and later renaming it back to same name.
        parentPath = Paths.get(metadataTableBasePath, HoodieTableMetaClient.METAFOLDER_NAME);
        metaFilePath = parentPath.resolve(metadataCompactionInstant + HoodieTimeline.COMMIT_EXTENSION);
        tempFilePath = FileCreateUtils.renameFileToTemp(metaFilePath, metadataCompactionInstant);
        validateMetadata(testTable);
        // this should retry the failed compaction in metadata table.
        doWriteOperation(testTable, "0000007", INSERT);
        validateMetadata(testTable);
        // add few more write and validate
        doWriteOperation(testTable, "0000008", INSERT);
        doWriteOperation(testTable, "0000009", UPSERT);
        validateMetadata(testTable);
    }
}

Example 4 with HoodieTableType

use of org.apache.hudi.common.model.HoodieTableType in project hudi by apache.

the class TestHoodieBackedMetadata method testManualRollbacks.

/**
 * Test that manual rollbacks work correctly and enough timeline history is maintained on the metadata table
 * timeline.
 */
@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testManualRollbacks(final boolean populateMateFields) throws Exception {
    HoodieTableType tableType = COPY_ON_WRITE;
    init(tableType, false);
    // Setting to archive more aggressively on the Metadata Table than the Dataset
    final int maxDeltaCommitsBeforeCompaction = 4;
    final int minArchiveCommitsMetadata = 2;
    final int minArchiveCommitsDataset = 4;
    writeConfig = getWriteConfigBuilder(true, true, false).withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).archiveCommitsWith(minArchiveCommitsMetadata, minArchiveCommitsMetadata + 1).retainCommits(1).withMaxNumDeltaCommitsBeforeCompaction(maxDeltaCommitsBeforeCompaction).withPopulateMetaFields(populateMateFields).build()).withCompactionConfig(HoodieCompactionConfig.newBuilder().archiveCommitsWith(minArchiveCommitsDataset, minArchiveCommitsDataset + 1).retainCommits(1).retainFileVersions(1).withAutoClean(false).withAsyncClean(true).build()).build();
    initWriteConfigAndMetatableWriter(writeConfig, true);
    doWriteInsertAndUpsert(testTable, "000001", "000002", false);
    for (int i = 3; i < 10; i++) {
        doWriteOperation(testTable, "00000" + i);
        archiveDataTable(writeConfig, metaClient);
    }
    validateMetadata(testTable);
    // We can only rollback those commits whose deltacommit have not been archived yet.
    int numRollbacks = 0;
    boolean exceptionRaised = false;
    List<HoodieInstant> allInstants = metaClient.reloadActiveTimeline().getCommitsTimeline().getReverseOrderedInstants().collect(Collectors.toList());
    for (HoodieInstant instantToRollback : allInstants) {
        try {
            testTable.doRollback(instantToRollback.getTimestamp(), String.valueOf(Time.now()));
            validateMetadata(testTable);
            ++numRollbacks;
        } catch (HoodieMetadataException e) {
            exceptionRaised = true;
            break;
        }
    }
    assertTrue(exceptionRaised, "Rollback of archived instants should fail");
    // Since each rollback also creates a deltacommit, we can only support rolling back of half of the original
    // instants present before rollback started.
    assertTrue(numRollbacks >= Math.max(minArchiveCommitsDataset, minArchiveCommitsMetadata) / 2, "Rollbacks of non archived instants should work");
}

Example 5 with HoodieTableType

use of org.apache.hudi.common.model.HoodieTableType in project hudi by apache.

the class TestHoodieDeltaStreamerWithMultiWriter method runJobsInParallel.

private void runJobsInParallel(String tableBasePath, HoodieTableType tableType, int totalRecords, HoodieDeltaStreamer ingestionJob, HoodieDeltaStreamer.Config cfgIngestionJob, HoodieDeltaStreamer backfillJob, HoodieDeltaStreamer.Config cfgBackfillJob, boolean expectConflict, String jobId) throws Exception {
    ExecutorService service = Executors.newFixedThreadPool(2);
    HoodieTableMetaClient meta = HoodieTableMetaClient.builder().setConf(hadoopConf()).setBasePath(tableBasePath).build();
    HoodieTimeline timeline = meta.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
    String lastSuccessfulCommit = timeline.lastInstant().get().getTimestamp();
    // Condition for parallel ingestion job
    Function<Boolean, Boolean> conditionForRegularIngestion = (r) -> {
        if (tableType.equals(HoodieTableType.MERGE_ON_READ)) {
            TestHoodieDeltaStreamer.TestHelpers.assertAtleastNDeltaCommitsAfterCommit(3, lastSuccessfulCommit, tableBasePath, fs());
        } else {
            TestHoodieDeltaStreamer.TestHelpers.assertAtleastNCompactionCommitsAfterCommit(3, lastSuccessfulCommit, tableBasePath, fs());
        }
        TestHoodieDeltaStreamer.TestHelpers.assertRecordCount(totalRecords, tableBasePath + "/*/*.parquet", sqlContext());
        TestHoodieDeltaStreamer.TestHelpers.assertDistanceCount(totalRecords, tableBasePath + "/*/*.parquet", sqlContext());
        return true;
    };
    AtomicBoolean continuousFailed = new AtomicBoolean(false);
    AtomicBoolean backfillFailed = new AtomicBoolean(false);
    try {
        Future regularIngestionJobFuture = service.submit(() -> {
            try {
                deltaStreamerTestRunner(ingestionJob, cfgIngestionJob, conditionForRegularIngestion, jobId);
            } catch (Throwable ex) {
                continuousFailed.set(true);
                LOG.error("Continuous job failed " + ex.getMessage());
                throw new RuntimeException(ex);
            }
        });
        Future backfillJobFuture = service.submit(() -> {
            try {
                // trigger backfill atleast after 1 requested entry is added to timeline from continuous job. If not, there is a chance that backfill will complete even before
                // continuous job starts.
                awaitCondition(new GetCommitsAfterInstant(tableBasePath, lastSuccessfulCommit));
                backfillJob.sync();
            } catch (Throwable ex) {
                LOG.error("Backfilling job failed " + ex.getMessage());
                backfillFailed.set(true);
                throw new RuntimeException(ex);
            }
        });
        backfillJobFuture.get();
        regularIngestionJobFuture.get();
        if (expectConflict) {
            Assertions.fail("Failed to handle concurrent writes");
        }
    } catch (Exception e) {
        /*
       * Need to perform getMessage().contains since the exception coming
       * from {@link org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.DeltaSyncService} gets wrapped many times into RuntimeExceptions.
       */
        if (expectConflict && e.getCause().getMessage().contains(ConcurrentModificationException.class.getName())) {
            // expected ConcurrentModificationException since ingestion & backfill will have overlapping writes
            if (backfillFailed.get()) {
                // if backfill job failed, shutdown the continuous job.
                LOG.warn("Calling shutdown on ingestion job since the backfill job has failed for " + jobId);
                ingestionJob.shutdownGracefully();
            }
        } else {
            LOG.error("Conflict happened, but not expected " + e.getCause().getMessage());
            throw e;
        }
    }
}