Examples with HoodieRollbackMetadata org.apache.hudi.avro.model.HoodieRollbackMetadata used on opensource projects

Example 11 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class BaseRestoreActionExecutor method finishRestore.

private HoodieRestoreMetadata finishRestore(Map<String, List<HoodieRollbackMetadata>> instantToMetadata, List<HoodieInstant> instantsRolledBack, long durationInMs) throws IOException {
    HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.convertRestoreMetadata(instantTime, durationInMs, instantsRolledBack, instantToMetadata);
    writeToMetadata(restoreMetadata);
    table.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, instantTime), TimelineMetadataUtils.serializeRestoreMetadata(restoreMetadata));
    // get all pending rollbacks instants after restore instant time and delete them.
    // if not, rollbacks will be considered not completed and might hinder metadata table compaction.
    List<HoodieInstant> instantsToRollback = table.getActiveTimeline().getRollbackTimeline().getReverseOrderedInstants().filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), restoreInstantTime)).collect(Collectors.toList());
    instantsToRollback.forEach(entry -> {
        table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
        table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
    });
    LOG.info("Commits " + instantsRolledBack + " rollback is complete. Restored table to " + restoreInstantTime);
    return restoreMetadata;
}

Example 12 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class BaseRollbackActionExecutor method runRollback.

private HoodieRollbackMetadata runRollback(HoodieTable<T, I, K, O> table, HoodieInstant rollbackInstant, HoodieRollbackPlan rollbackPlan) {
    ValidationUtils.checkArgument(rollbackInstant.getState().equals(HoodieInstant.State.REQUESTED) || rollbackInstant.getState().equals(HoodieInstant.State.INFLIGHT));
    final HoodieTimer timer = new HoodieTimer();
    timer.startTimer();
    final HoodieInstant inflightInstant = rollbackInstant.isRequested() ? table.getActiveTimeline().transitionRollbackRequestedToInflight(rollbackInstant) : rollbackInstant;
    HoodieTimer rollbackTimer = new HoodieTimer().startTimer();
    List<HoodieRollbackStat> stats = doRollbackAndGetStats(rollbackPlan);
    HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.convertRollbackMetadata(instantTime, Option.of(rollbackTimer.endTimer()), Collections.singletonList(instantToRollback), stats);
    if (!skipTimelinePublish) {
        finishRollback(inflightInstant, rollbackMetadata);
    }
    // Finally, remove the markers post rollback.
    WriteMarkersFactory.get(config.getMarkersType(), table, instantToRollback.getTimestamp()).quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
    return rollbackMetadata;
}

Example 13 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class TestMetadataConversionUtils method createRollbackMetadata.

private void createRollbackMetadata(String instantTime) throws Exception {
    HoodieRollbackMetadata rollbackMetadata = new HoodieRollbackMetadata();
    rollbackMetadata.setCommitsRollback(Arrays.asList(instantTime));
    rollbackMetadata.setStartRollbackTime(instantTime);
    HoodieRollbackPartitionMetadata rollbackPartitionMetadata = new HoodieRollbackPartitionMetadata();
    rollbackPartitionMetadata.setPartitionPath("p1");
    rollbackPartitionMetadata.setSuccessDeleteFiles(Arrays.asList("f1"));
    rollbackPartitionMetadata.setFailedDeleteFiles(new ArrayList<>());
    rollbackPartitionMetadata.setRollbackLogFiles(new HashMap<>());
    Map<String, HoodieRollbackPartitionMetadata> partitionMetadataMap = new HashMap<>();
    partitionMetadataMap.put("p1", rollbackPartitionMetadata);
    rollbackMetadata.setPartitionMetadata(partitionMetadataMap);
    rollbackMetadata.setInstantsRollback(Arrays.asList(new HoodieInstantInfo("1", HoodieTimeline.COMMIT_ACTION)));
    HoodieTestTable.of(metaClient).addRollback(instantTime, rollbackMetadata);
}

Example 14 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class HoodieTestTable method doRollbackWithExtraFiles.

public HoodieTestTable doRollbackWithExtraFiles(String commitTimeToRollback, String commitTime, Map<String, List<String>> extraFiles) throws Exception {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    Option<HoodieCommitMetadata> commitMetadata = getMetadataForInstant(commitTimeToRollback);
    if (!commitMetadata.isPresent()) {
        throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitTimeToRollback);
    }
    Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
    for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
        deleteFilesInPartition(entry.getKey(), entry.getValue());
    }
    for (Map.Entry<String, List<String>> entry : extraFiles.entrySet()) {
        if (partitionFiles.containsKey(entry.getKey())) {
            partitionFiles.get(entry.getKey()).addAll(entry.getValue());
        }
    }
    HoodieRollbackMetadata rollbackMetadata = getRollbackMetadata(commitTimeToRollback, partitionFiles);
    return addRollback(commitTime, rollbackMetadata);
}

Example 15 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class TimelineMetadataUtils method convertRollbackMetadata.

public static HoodieRollbackMetadata convertRollbackMetadata(String startRollbackTime, Option<Long> durationInMs, List<HoodieInstant> instants, List<HoodieRollbackStat> rollbackStats) {
    Map<String, HoodieRollbackPartitionMetadata> partitionMetadataBuilder = new HashMap<>();
    int totalDeleted = 0;
    for (HoodieRollbackStat stat : rollbackStats) {
        Map<String, Long> rollbackLogFiles = stat.getCommandBlocksCount().keySet().stream().collect(Collectors.toMap(f -> f.getPath().toString(), FileStatus::getLen));
        HoodieRollbackPartitionMetadata metadata = new HoodieRollbackPartitionMetadata(stat.getPartitionPath(), stat.getSuccessDeleteFiles(), stat.getFailedDeleteFiles(), rollbackLogFiles);
        partitionMetadataBuilder.put(stat.getPartitionPath(), metadata);
        totalDeleted += stat.getSuccessDeleteFiles().size();
    }
    return new HoodieRollbackMetadata(startRollbackTime, durationInMs.orElseGet(() -> -1L), totalDeleted, instants.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()), Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION, instants.stream().map(instant -> new HoodieInstantInfo(instant.getTimestamp(), instant.getAction())).collect(Collectors.toList()));
}