use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.
the class BaseHoodieWriteClient method restoreToInstant.
/**
* NOTE : This action requires all writers (ingest and compact) to a table to be stopped before proceeding. Revert
* the (inflight/committed) record changes for all commits after the provided instant time.
*
* @param instantTime Instant time to which restoration is requested
*/
public HoodieRestoreMetadata restoreToInstant(final String instantTime) throws HoodieRestoreException {
LOG.info("Begin restore to instant " + instantTime);
final String restoreInstantTime = HoodieActiveTimeline.createNewInstantTime();
Timer.Context timerContext = metrics.getRollbackCtx();
try {
HoodieTable<T, I, K, O> table = initTable(WriteOperationType.UNKNOWN, Option.empty());
Option<HoodieRestorePlan> restorePlanOption = table.scheduleRestore(context, restoreInstantTime, instantTime);
if (restorePlanOption.isPresent()) {
HoodieRestoreMetadata restoreMetadata = table.restore(context, restoreInstantTime, instantTime);
if (timerContext != null) {
final long durationInMs = metrics.getDurationInMs(timerContext.stop());
final long totalFilesDeleted = restoreMetadata.getHoodieRestoreMetadata().values().stream().flatMap(Collection::stream).mapToLong(HoodieRollbackMetadata::getTotalFilesDeleted).sum();
metrics.updateRollbackMetrics(durationInMs, totalFilesDeleted);
}
return restoreMetadata;
} else {
throw new HoodieRestoreException("Failed to restore " + config.getBasePath() + " to commit " + instantTime);
}
} catch (Exception e) {
throw new HoodieRestoreException("Failed to restore to " + instantTime, e);
}
}
use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.
the class IncrementalTimelineSyncFileSystemView method addRestoreInstant.
/**
* Add newly found restore instant.
*
* @param timeline Hoodie Timeline
* @param instant Restore Instant
*/
private void addRestoreInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException {
LOG.info("Syncing restore instant (" + instant + ")");
HoodieRestoreMetadata metadata = TimelineMetadataUtils.deserializeAvroMetadata(timeline.getInstantDetails(instant).get(), HoodieRestoreMetadata.class);
Map<String, List<Pair<String, String>>> partitionFiles = metadata.getHoodieRestoreMetadata().entrySet().stream().flatMap(entry -> {
return entry.getValue().stream().flatMap(e -> e.getPartitionMetadata().entrySet().stream().flatMap(e2 -> {
return e2.getValue().getSuccessDeleteFiles().stream().map(x -> Pair.of(e2.getKey(), x));
}));
}).collect(Collectors.groupingBy(Pair::getKey));
partitionFiles.entrySet().stream().forEach(e -> {
removeFileSlicesForPartition(timeline, instant, e.getKey(), e.getValue().stream().map(x -> x.getValue()).collect(Collectors.toList()));
});
if (metadata.getRestoreInstantInfo() != null) {
Set<String> rolledbackInstants = metadata.getRestoreInstantInfo().stream().filter(instantInfo -> HoodieTimeline.REPLACE_COMMIT_ACTION.equals(instantInfo.getAction())).map(instantInfo -> instantInfo.getCommitTime()).collect(Collectors.toSet());
removeReplacedFileIdsAtInstants(rolledbackInstants);
}
LOG.info("Done Syncing restore instant (" + instant + ")");
}
use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.
the class TestTimelineUtils method getRestoreMetadata.
private byte[] getRestoreMetadata(String basePath, String partition, String commitTs, int count, String actionType) throws IOException {
List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
rollbackM.add(getRollbackMetadataInstance(basePath, partition, commitTs, count, actionType));
List<HoodieInstant> rollbackInstants = new ArrayList<>();
rollbackInstants.add(new HoodieInstant(false, commitTs, actionType));
HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(commitTs, 200, rollbackInstants, CollectionUtils.createImmutableMap(commitTs, rollbackM));
return TimelineMetadataUtils.serializeRestoreMetadata(metadata).get();
}
use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.
the class HoodieBackedTableMetadata method getRollbackedCommits.
/**
* Returns a list of commits which were rolled back as part of a Rollback or Restore operation.
*
* @param instant The Rollback operation to read
* @param timeline instant of timeline from dataset.
*/
private List<String> getRollbackedCommits(HoodieInstant instant, HoodieActiveTimeline timeline) {
try {
if (instant.getAction().equals(HoodieTimeline.ROLLBACK_ACTION)) {
HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.deserializeHoodieRollbackMetadata(timeline.getInstantDetails(instant).get());
return rollbackMetadata.getCommitsRollback();
}
List<String> rollbackedCommits = new LinkedList<>();
if (instant.getAction().equals(HoodieTimeline.RESTORE_ACTION)) {
// Restore is made up of several rollbacks
HoodieRestoreMetadata restoreMetadata = TimelineMetadataUtils.deserializeHoodieRestoreMetadata(timeline.getInstantDetails(instant).get());
restoreMetadata.getHoodieRestoreMetadata().values().forEach(rms -> {
rms.forEach(rm -> rollbackedCommits.addAll(rm.getCommitsRollback()));
});
}
return rollbackedCommits;
} catch (IOException e) {
throw new HoodieMetadataException("Error retrieving rollback commits for instant " + instant, e);
}
}
use of org.apache.hudi.avro.model.HoodieRestoreMetadata in project hudi by apache.
the class BaseRestoreActionExecutor method execute.
@Override
public HoodieRestoreMetadata execute() {
HoodieTimer restoreTimer = new HoodieTimer();
restoreTimer.startTimer();
Option<HoodieInstant> restoreInstant = table.getRestoreTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
if (!restoreInstant.isPresent()) {
throw new HoodieRollbackException("No pending restore instants found to execute restore");
}
try {
List<HoodieInstant> instantsToRollback = getInstantsToRollback(restoreInstant.get());
ValidationUtils.checkArgument(restoreInstant.get().getState().equals(HoodieInstant.State.REQUESTED) || restoreInstant.get().getState().equals(HoodieInstant.State.INFLIGHT));
Map<String, List<HoodieRollbackMetadata>> instantToMetadata = new HashMap<>();
if (restoreInstant.get().isRequested()) {
table.getActiveTimeline().transitionRestoreRequestedToInflight(restoreInstant.get());
}
instantsToRollback.forEach(instant -> {
instantToMetadata.put(instant.getTimestamp(), Collections.singletonList(rollbackInstant(instant)));
LOG.info("Deleted instant " + instant);
});
return finishRestore(instantToMetadata, instantsToRollback, restoreTimer.endTimer());
} catch (IOException io) {
throw new HoodieRestoreException("unable to Restore instant " + restoreInstant.get(), io);
}
}
Aggregations