Search in sources :

Example 1 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class BaseHoodieWriteClient method restoreToInstant.

/**
 * NOTE : This action requires all writers (ingest and compact) to a table to be stopped before proceeding. Revert
 * the (inflight/committed) record changes for all commits after the provided instant time.
 *
 * @param instantTime Instant time to which restoration is requested
 */
public HoodieRestoreMetadata restoreToInstant(final String instantTime) throws HoodieRestoreException {
    LOG.info("Begin restore to instant " + instantTime);
    final String restoreInstantTime = HoodieActiveTimeline.createNewInstantTime();
    Timer.Context timerContext = metrics.getRollbackCtx();
    try {
        HoodieTable<T, I, K, O> table = initTable(WriteOperationType.UNKNOWN, Option.empty());
        Option<HoodieRestorePlan> restorePlanOption = table.scheduleRestore(context, restoreInstantTime, instantTime);
        if (restorePlanOption.isPresent()) {
            HoodieRestoreMetadata restoreMetadata = table.restore(context, restoreInstantTime, instantTime);
            if (timerContext != null) {
                final long durationInMs = metrics.getDurationInMs(timerContext.stop());
                final long totalFilesDeleted = restoreMetadata.getHoodieRestoreMetadata().values().stream().flatMap(Collection::stream).mapToLong(HoodieRollbackMetadata::getTotalFilesDeleted).sum();
                metrics.updateRollbackMetrics(durationInMs, totalFilesDeleted);
            }
            return restoreMetadata;
        } else {
            throw new HoodieRestoreException("Failed to restore " + config.getBasePath() + " to commit " + instantTime);
        }
    } catch (Exception e) {
        throw new HoodieRestoreException("Failed to restore to " + instantTime, e);
    }
}
Also used : HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) HoodieRestorePlan(org.apache.hudi.avro.model.HoodieRestorePlan) HoodieException(org.apache.hudi.exception.HoodieException) ParseException(java.text.ParseException) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) HoodieCommitException(org.apache.hudi.exception.HoodieCommitException) HoodieSavepointException(org.apache.hudi.exception.HoodieSavepointException) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) Timer(com.codahale.metrics.Timer) Collection(java.util.Collection) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException)

Example 2 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class IncrementalTimelineSyncFileSystemView method addRestoreInstant.

/**
 * Add newly found restore instant.
 *
 * @param timeline Hoodie Timeline
 * @param instant Restore Instant
 */
private void addRestoreInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException {
    LOG.info("Syncing restore instant (" + instant + ")");
    HoodieRestoreMetadata metadata = TimelineMetadataUtils.deserializeAvroMetadata(timeline.getInstantDetails(instant).get(), HoodieRestoreMetadata.class);
    Map<String, List<Pair<String, String>>> partitionFiles = metadata.getHoodieRestoreMetadata().entrySet().stream().flatMap(entry -> {
        return entry.getValue().stream().flatMap(e -> e.getPartitionMetadata().entrySet().stream().flatMap(e2 -> {
            return e2.getValue().getSuccessDeleteFiles().stream().map(x -> Pair.of(e2.getKey(), x));
        }));
    }).collect(Collectors.groupingBy(Pair::getKey));
    partitionFiles.entrySet().stream().forEach(e -> {
        removeFileSlicesForPartition(timeline, instant, e.getKey(), e.getValue().stream().map(x -> x.getValue()).collect(Collectors.toList()));
    });
    if (metadata.getRestoreInstantInfo() != null) {
        Set<String> rolledbackInstants = metadata.getRestoreInstantInfo().stream().filter(instantInfo -> HoodieTimeline.REPLACE_COMMIT_ACTION.equals(instantInfo.getAction())).map(instantInfo -> instantInfo.getCommitTime()).collect(Collectors.toSet());
        removeReplacedFileIdsAtInstants(rolledbackInstants);
    }
    LOG.info("Done Syncing restore instant (" + instant + ")");
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) FileSlice(org.apache.hudi.common.model.FileSlice) TimelineDiffHelper(org.apache.hudi.common.table.timeline.TimelineDiffHelper) HoodieException(org.apache.hudi.exception.HoodieException) Option(org.apache.hudi.common.util.Option) FileStatus(org.apache.hadoop.fs.FileStatus) Logger(org.apache.log4j.Logger) HoodieFileGroup(org.apache.hudi.common.model.HoodieFileGroup) CleanerUtils(org.apache.hudi.common.util.CleanerUtils) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) Path(org.apache.hadoop.fs.Path) HoodieLogFile(org.apache.hudi.common.model.HoodieLogFile) HoodieFileGroupId(org.apache.hudi.common.model.HoodieFileGroupId) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) Set(java.util.Set) HoodieCommitMetadata(org.apache.hudi.common.model.HoodieCommitMetadata) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) CompactionOperation(org.apache.hudi.common.model.CompactionOperation) HoodieReplaceCommitMetadata(org.apache.hudi.common.model.HoodieReplaceCommitMetadata) HoodieBaseFile(org.apache.hudi.common.model.HoodieBaseFile) List(java.util.List) HoodieCleanMetadata(org.apache.hudi.avro.model.HoodieCleanMetadata) TimelineDiffResult(org.apache.hudi.common.table.timeline.TimelineDiffHelper.TimelineDiffResult) HoodieWriteStat(org.apache.hudi.common.model.HoodieWriteStat) HoodieCompactionPlan(org.apache.hudi.avro.model.HoodieCompactionPlan) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) LogManager(org.apache.log4j.LogManager) FSUtils(org.apache.hudi.common.fs.FSUtils) CompactionUtils(org.apache.hudi.common.util.CompactionUtils) Pair(org.apache.hudi.common.util.collection.Pair) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) List(java.util.List)

Example 3 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class TestTimelineUtils method getRestoreMetadata.

private byte[] getRestoreMetadata(String basePath, String partition, String commitTs, int count, String actionType) throws IOException {
    List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
    rollbackM.add(getRollbackMetadataInstance(basePath, partition, commitTs, count, actionType));
    List<HoodieInstant> rollbackInstants = new ArrayList<>();
    rollbackInstants.add(new HoodieInstant(false, commitTs, actionType));
    HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(commitTs, 200, rollbackInstants, CollectionUtils.createImmutableMap(commitTs, rollbackM));
    return TimelineMetadataUtils.serializeRestoreMetadata(metadata).get();
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) ArrayList(java.util.ArrayList) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata)

Example 4 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class HoodieBackedTableMetadata method getRollbackedCommits.

/**
 * Returns a list of commits which were rolled back as part of a Rollback or Restore operation.
 *
 * @param instant  The Rollback operation to read
 * @param timeline instant of timeline from dataset.
 */
private List<String> getRollbackedCommits(HoodieInstant instant, HoodieActiveTimeline timeline) {
    try {
        if (instant.getAction().equals(HoodieTimeline.ROLLBACK_ACTION)) {
            HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.deserializeHoodieRollbackMetadata(timeline.getInstantDetails(instant).get());
            return rollbackMetadata.getCommitsRollback();
        }
        List<String> rollbackedCommits = new LinkedList<>();
        if (instant.getAction().equals(HoodieTimeline.RESTORE_ACTION)) {
            // Restore is made up of several rollbacks
            HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.deserializeHoodieRestoreMetadata(timeline.getInstantDetails(instant).get());
            restoreMetadata.getHoodieRestoreMetadata().values().forEach(rms -> {
                rms.forEach(rm -> rollbackedCommits.addAll(rm.getCommitsRollback()));
            });
        }
        return rollbackedCommits;
    } catch (IOException e) {
        throw new HoodieMetadataException("Error retrieving rollback commits for instant " + instant, e);
    }
}
Also used : HoodieMetadataException(org.apache.hudi.exception.HoodieMetadataException) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) LinkedList(java.util.LinkedList)

Example 5 with HoodieRestoreMetadata

use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.

the class BaseRestoreActionExecutor method execute.

@Override
public HoodieRestoreMetadata execute() {
    HoodieTimer restoreTimer = new HoodieTimer();
    restoreTimer.startTimer();
    Option<HoodieInstant> restoreInstant = table.getRestoreTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
    if (!restoreInstant.isPresent()) {
        throw new HoodieRollbackException("No pending restore instants found to execute restore");
    }
    try {
        List<HoodieInstant> instantsToRollback = getInstantsToRollback(restoreInstant.get());
        ValidationUtils.checkArgument(restoreInstant.get().getState().equals(HoodieInstant.State.REQUESTED) || restoreInstant.get().getState().equals(HoodieInstant.State.INFLIGHT));
        Map<String, List<HoodieRollbackMetadata>> instantToMetadata = new HashMap<>();
        if (restoreInstant.get().isRequested()) {
            table.getActiveTimeline().transitionRestoreRequestedToInflight(restoreInstant.get());
        }
        instantsToRollback.forEach(instant -> {
            instantToMetadata.put(instant.getTimestamp(), Collections.singletonList(rollbackInstant(instant)));
            LOG.info("Deleted instant " + instant);
        });
        return finishRestore(instantToMetadata, instantsToRollback, restoreTimer.endTimer());
    } catch (IOException io) {
        throw new HoodieRestoreException("unable to Restore instant " + restoreInstant.get(), io);
    }
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieTable(org.apache.hudi.table.HoodieTable) BaseActionExecutor(org.apache.hudi.table.action.BaseActionExecutor) HoodieRestorePlan(org.apache.hudi.avro.model.HoodieRestorePlan) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) Option(org.apache.hudi.common.util.Option) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) TransactionManager(org.apache.hudi.client.transaction.TransactionManager) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) HoodieInstantInfo(org.apache.hudi.avro.model.HoodieInstantInfo) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) List(java.util.List) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) LogManager(org.apache.log4j.LogManager) Collections(java.util.Collections) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HashMap(java.util.HashMap) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) List(java.util.List) IOException(java.io.IOException) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException)

Aggregations

HoodieRestoreMetadata (org.apache.hudi.avro.model.HoodieRestoreMetadata)8 IOException (java.io.IOException)6 HoodieRollbackMetadata (org.apache.hudi.avro.model.HoodieRollbackMetadata)6 HoodieInstant (org.apache.hudi.common.table.timeline.HoodieInstant)6 ArrayList (java.util.ArrayList)5 List (java.util.List)5 Map (java.util.Map)5 HashMap (java.util.HashMap)4 Collectors (java.util.stream.Collectors)4 HoodieTimeline (org.apache.hudi.common.table.timeline.HoodieTimeline)4 TimelineMetadataUtils (org.apache.hudi.common.table.timeline.TimelineMetadataUtils)4 Option (org.apache.hudi.common.util.Option)4 LogManager (org.apache.log4j.LogManager)4 Logger (org.apache.log4j.Logger)4 Collections (java.util.Collections)3 HoodieRestorePlan (org.apache.hudi.avro.model.HoodieRestorePlan)3 HoodieCommitMetadata (org.apache.hudi.common.model.HoodieCommitMetadata)3 Set (java.util.Set)2 Path (org.apache.hadoop.fs.Path)2 HoodieCleanMetadata (org.apache.hudi.avro.model.HoodieCleanMetadata)2