Examples with TimelineDiffResult org.apache.hudi.common.table.timeline.TimelineDiffHelper.TimelineDiffResult used on opensource projects

Example 1 with TimelineDiffResult

use of org.apache.hudi.common.table.timeline.TimelineDiffHelper.TimelineDiffResult in project hudi by apache.

the class IncrementalTimelineSyncFileSystemView method runSync.

@Override
protected void runSync(HoodieTimeline oldTimeline, HoodieTimeline newTimeline) {
    try {
        if (incrementalTimelineSyncEnabled) {
            TimelineDiffResult diffResult = TimelineDiffHelper.getNewInstantsForIncrementalSync(oldTimeline, newTimeline);
            if (diffResult.canSyncIncrementally()) {
                LOG.info("Doing incremental sync");
                runIncrementalSync(newTimeline, diffResult);
                LOG.info("Finished incremental sync");
                // Reset timeline to latest
                refreshTimeline(newTimeline);
                return;
            }
        }
    } catch (Exception ioe) {
        LOG.error("Got exception trying to perform incremental sync. Reverting to complete sync", ioe);
    }
    super.runSync(oldTimeline, newTimeline);
}

Example 2 with TimelineDiffResult

use of org.apache.hudi.common.table.timeline.TimelineDiffHelper.TimelineDiffResult in project hudi by apache.

the class IncrementalTimelineSyncFileSystemView method runIncrementalSync.

/**
 * Run incremental sync based on the diff result produced.
 *
 * @param timeline New Timeline
 * @param diffResult Timeline Diff Result
 */
private void runIncrementalSync(HoodieTimeline timeline, TimelineDiffResult diffResult) {
    LOG.info("Timeline Diff Result is :" + diffResult);
    // First remove pending compaction instants which were completed
    diffResult.getFinishedCompactionInstants().stream().forEach(instant -> {
        try {
            removePendingCompactionInstant(timeline, instant);
        } catch (IOException e) {
            throw new HoodieException(e);
        }
    });
    // Add new completed instants found in the latest timeline
    diffResult.getNewlySeenInstants().stream().filter(instant -> instant.isCompleted() || instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)).forEach(instant -> {
        try {
            if (instant.getAction().equals(HoodieTimeline.COMMIT_ACTION) || instant.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION)) {
                addCommitInstant(timeline, instant);
            } else if (instant.getAction().equals(HoodieTimeline.RESTORE_ACTION)) {
                addRestoreInstant(timeline, instant);
            } else if (instant.getAction().equals(HoodieTimeline.CLEAN_ACTION)) {
                addCleanInstant(timeline, instant);
            } else if (instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)) {
                addPendingCompactionInstant(timeline, instant);
            } else if (instant.getAction().equals(HoodieTimeline.ROLLBACK_ACTION)) {
                addRollbackInstant(timeline, instant);
            } else if (instant.getAction().equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
                addReplaceInstant(timeline, instant);
            }
        } catch (IOException ioe) {
            throw new HoodieException(ioe);
        }
    });
}