Examples with HoodieRestoreMetadata org.apache.hudi.avro.model.HoodieRestoreMetadata used on opensource projects

Example 6 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class BaseRestoreActionExecutor method finishRestore.

private HoodieRestoreMetadata finishRestore(Map<String, List<HoodieRollbackMetadata>> instantToMetadata, List<HoodieInstant> instantsRolledBack, long durationInMs) throws IOException {
    HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.convertRestoreMetadata(instantTime, durationInMs, instantsRolledBack, instantToMetadata);
    writeToMetadata(restoreMetadata);
    table.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, instantTime), TimelineMetadataUtils.serializeRestoreMetadata(restoreMetadata));
    // get all pending rollbacks instants after restore instant time and delete them.
    // if not, rollbacks will be considered not completed and might hinder metadata table compaction.
    List<HoodieInstant> instantsToRollback = table.getActiveTimeline().getRollbackTimeline().getReverseOrderedInstants().filter(instant -> HoodieActiveTimeline.GREATER_THAN.test(instant.getTimestamp(), restoreInstantTime)).collect(Collectors.toList());
    instantsToRollback.forEach(entry -> {
        table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
        table.getActiveTimeline().deletePending(new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, entry.getTimestamp()));
    });
    LOG.info("Commits " + instantsRolledBack + " rollback is complete. Restored table to " + restoreInstantTime);
    return restoreMetadata;
}

Example 7 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class HoodieTestTable method doRestore.

public HoodieTestTable doRestore(String commitToRestoreTo, String restoreTime) throws Exception {
    metaClient = HoodieTableMetaClient.reload(metaClient);
    List<HoodieInstant> commitsToRollback = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().findInstantsAfter(commitToRestoreTo).getReverseOrderedInstants().collect(Collectors.toList());
    Map<String, List<HoodieRollbackMetadata>> rollbackMetadataMap = new HashMap<>();
    for (HoodieInstant commitInstantToRollback : commitsToRollback) {
        Option<HoodieCommitMetadata> commitMetadata = getCommitMeta(commitInstantToRollback);
        if (!commitMetadata.isPresent()) {
            throw new IllegalArgumentException("Instant to rollback not present in timeline: " + commitInstantToRollback.getTimestamp());
        }
        Map<String, List<String>> partitionFiles = getPartitionFiles(commitMetadata.get());
        rollbackMetadataMap.put(commitInstantToRollback.getTimestamp(), Collections.singletonList(getRollbackMetadata(commitInstantToRollback.getTimestamp(), partitionFiles)));
        for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
            deleteFilesInPartition(entry.getKey(), entry.getValue());
        }
    }
    HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.convertRestoreMetadata(restoreTime, 1000L, commitsToRollback, rollbackMetadataMap);
    return addRestore(restoreTime, restoreMetadata);
}

Example 8 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class TestIncrementalFSViewSync method performRestore.

/**
 * Simulate Restore of an instant in timeline and fsview.
 *
 * @param instant Instant to be rolled-back
 * @param files List of files to be deleted as part of rollback
 * @param rollbackInstant Restore Instant
 */
private void performRestore(HoodieInstant instant, List<String> files, String rollbackInstant, boolean isRestore) throws IOException {
    Map<String, List<String>> partititonToFiles = deleteFiles(files);
    List<HoodieRollbackStat> rollbackStats = partititonToFiles.entrySet().stream().map(e -> new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>())).collect(Collectors.toList());
    List<HoodieInstant> rollbacks = new ArrayList<>();
    rollbacks.add(instant);
    HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.convertRollbackMetadata(rollbackInstant, Option.empty(), rollbacks, rollbackStats);
    if (isRestore) {
        List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
        rollbackM.add(rollbackMetadata);
        HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(rollbackInstant, 100, Collections.singletonList(instant), CollectionUtils.createImmutableMap(rollbackInstant, rollbackM));
        HoodieInstant restoreInstant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, rollbackInstant);
        metaClient.getActiveTimeline().createNewInstant(restoreInstant);
        metaClient.getActiveTimeline().saveAsComplete(restoreInstant, TimelineMetadataUtils.serializeRestoreMetadata(metadata));
    } else {
        metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant));
        metaClient.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant), TimelineMetadataUtils.serializeRollbackMetadata(rollbackMetadata));
    }
    boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
    assertTrue(deleted);
}