Examples with HoodieRollbackStat org.apache.hudi.common.HoodieRollbackStat used on opensource projects

Example 6 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class MergeOnReadRollbackActionExecutor method executeRollback.

@Override
protected List<HoodieRollbackStat> executeRollback(HoodieRollbackPlan hoodieRollbackPlan) {
    HoodieTimer rollbackTimer = new HoodieTimer();
    rollbackTimer.startTimer();
    LOG.info("Rolling back instant " + instantToRollback);
    HoodieInstant resolvedInstant = instantToRollback;
    // Atomically un-publish all non-inflight commits
    if (instantToRollback.isCompleted()) {
        LOG.info("Un-publishing instant " + instantToRollback + ", deleteInstants=" + deleteInstants);
        resolvedInstant = table.getActiveTimeline().revertToInflight(instantToRollback);
        // reload meta-client to reflect latest timeline status
        table.getMetaClient().reloadActiveTimeline();
    }
    List<HoodieRollbackStat> allRollbackStats = new ArrayList<>();
    // deleting the timeline file
    if (!resolvedInstant.isRequested()) {
        LOG.info("Unpublished " + resolvedInstant);
        allRollbackStats = executeRollback(instantToRollback, hoodieRollbackPlan);
    }
    dropBootstrapIndexIfNeeded(resolvedInstant);
    // Delete Inflight instants if enabled
    deleteInflightAndRequestedInstant(deleteInstants, table.getActiveTimeline(), resolvedInstant);
    LOG.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer());
    return allRollbackStats;
}

Example 7 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class ZeroToOneUpgradeHandler method recreateMarkers.

/**
 * Recreate markers in new format.
 * Step1: Delete existing markers
 * Step2: Collect all rollback file info.
 * Step3: recreate markers for all interested files.
 *
 * @param commitInstantTime instant of interest for which markers need to be recreated.
 * @param table             instance of {@link HoodieTable} to use
 * @param context           instance of {@link HoodieEngineContext} to use
 * @throws HoodieRollbackException on any exception during upgrade.
 */
protected void recreateMarkers(final String commitInstantTime, HoodieTable table, HoodieEngineContext context, int parallelism) throws HoodieRollbackException {
    try {
        // fetch hoodie instant
        Option<HoodieInstant> commitInstantOpt = Option.fromJavaOptional(table.getActiveTimeline().getCommitsTimeline().getInstants().filter(instant -> HoodieActiveTimeline.EQUALS.test(instant.getTimestamp(), commitInstantTime)).findFirst());
        if (commitInstantOpt.isPresent()) {
            // delete existing markers
            WriteMarkers writeMarkers = WriteMarkersFactory.get(MarkerType.DIRECT, table, commitInstantTime);
            writeMarkers.quietDeleteMarkerDir(context, parallelism);
            // generate rollback stats
            List<ListingBasedRollbackRequest> rollbackRequests;
            if (table.getMetaClient().getTableType() == HoodieTableType.COPY_ON_WRITE) {
                rollbackRequests = RollbackUtils.generateRollbackRequestsByListingCOW(context, table.getMetaClient().getBasePath());
            } else {
                rollbackRequests = RollbackUtils.generateRollbackRequestsUsingFileListingMOR(commitInstantOpt.get(), table, context);
            }
            List<HoodieRollbackStat> rollbackStats = getListBasedRollBackStats(table.getMetaClient(), table.getConfig(), context, commitInstantOpt, rollbackRequests);
            // recreate markers adhering to marker based rollback
            for (HoodieRollbackStat rollbackStat : rollbackStats) {
                for (String path : rollbackStat.getSuccessDeleteFiles()) {
                    String dataFileName = path.substring(path.lastIndexOf("/") + 1);
                    // not feasible to differentiate MERGE from CREATE. hence creating with MERGE IOType for all base files.
                    writeMarkers.create(rollbackStat.getPartitionPath(), dataFileName, IOType.MERGE);
                }
                for (FileStatus fileStatus : rollbackStat.getCommandBlocksCount().keySet()) {
                    writeMarkers.create(rollbackStat.getPartitionPath(), getFileNameForMarkerFromLogFile(fileStatus.getPath().toString(), table), IOType.APPEND);
                }
            }
        }
    } catch (Exception e) {
        throw new HoodieRollbackException("Exception thrown while upgrading Hoodie Table from version 0 to 1", e);
    }
}

Example 8 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class BaseRollbackActionExecutor method doRollbackAndGetStats.

public List<HoodieRollbackStat> doRollbackAndGetStats(HoodieRollbackPlan hoodieRollbackPlan) {
    final String instantTimeToRollback = instantToRollback.getTimestamp();
    final boolean isPendingCompaction = Objects.equals(HoodieTimeline.COMPACTION_ACTION, instantToRollback.getAction()) && !instantToRollback.isCompleted();
    final boolean isPendingClustering = Objects.equals(HoodieTimeline.REPLACE_COMMIT_ACTION, instantToRollback.getAction()) && !instantToRollback.isCompleted() && ClusteringUtils.getClusteringPlan(table.getMetaClient(), instantToRollback).isPresent();
    validateSavepointRollbacks();
    if (!isPendingCompaction && !isPendingClustering) {
        validateRollbackCommitSequence();
    }
    try {
        List<HoodieRollbackStat> stats = executeRollback(hoodieRollbackPlan);
        LOG.info("Rolled back inflight instant " + instantTimeToRollback);
        if (!isPendingCompaction) {
            rollBackIndex();
        }
        return stats;
    } catch (IOException e) {
        throw new HoodieIOException("Unable to execute rollback ", e);
    }
}

Example 9 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class BaseRollbackActionExecutor method runRollback.

private HoodieRollbackMetadata runRollback(HoodieTable<T, I, K, O> table, HoodieInstant rollbackInstant, HoodieRollbackPlan rollbackPlan) {
    ValidationUtils.checkArgument(rollbackInstant.getState().equals(HoodieInstant.State.REQUESTED) || rollbackInstant.getState().equals(HoodieInstant.State.INFLIGHT));
    final HoodieTimer timer = new HoodieTimer();
    timer.startTimer();
    final HoodieInstant inflightInstant = rollbackInstant.isRequested() ? table.getActiveTimeline().transitionRollbackRequestedToInflight(rollbackInstant) : rollbackInstant;
    HoodieTimer rollbackTimer = new HoodieTimer().startTimer();
    List<HoodieRollbackStat> stats = doRollbackAndGetStats(rollbackPlan);
    HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.convertRollbackMetadata(instantTime, Option.of(rollbackTimer.endTimer()), Collections.singletonList(instantToRollback), stats);
    if (!skipTimelinePublish) {
        finishRollback(inflightInstant, rollbackMetadata);
    }
    // Finally, remove the markers post rollback.
    WriteMarkersFactory.get(config.getMarkersType(), table, instantToRollback.getTimestamp()).quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
    return rollbackMetadata;
}

Example 10 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class CopyOnWriteRollbackActionExecutor method executeRollback.

@Override
protected List<HoodieRollbackStat> executeRollback(HoodieRollbackPlan hoodieRollbackPlan) {
    HoodieTimer rollbackTimer = new HoodieTimer();
    rollbackTimer.startTimer();
    List<HoodieRollbackStat> stats = new ArrayList<>();
    HoodieActiveTimeline activeTimeline = table.getActiveTimeline();
    HoodieInstant resolvedInstant = instantToRollback;
    if (instantToRollback.isCompleted()) {
        LOG.info("Unpublishing instant " + instantToRollback);
        resolvedInstant = activeTimeline.revertToInflight(instantToRollback);
        // reload meta-client to reflect latest timeline status
        table.getMetaClient().reloadActiveTimeline();
    }
    // deleting the timeline file
    if (!resolvedInstant.isRequested()) {
        // delete all the data files for this commit
        LOG.info("Clean out all base files generated for commit: " + resolvedInstant);
        stats = executeRollback(resolvedInstant, hoodieRollbackPlan);
    }
    dropBootstrapIndexIfNeeded(instantToRollback);
    // Delete Inflight instant if enabled
    deleteInflightAndRequestedInstant(deleteInstants, activeTimeline, resolvedInstant);
    LOG.info("Time(in ms) taken to finish rollback " + rollbackTimer.endTimer());
    return stats;
}