Search in sources :

Example 1 with HoodieRollbackException

use of org.apache.hudi.exception.HoodieRollbackException in project hudi by apache.

the class BaseRestoreActionExecutor method execute.

@Override
public HoodieRestoreMetadata execute() {
    HoodieTimer restoreTimer = new HoodieTimer();
    restoreTimer.startTimer();
    Option<HoodieInstant> restoreInstant = table.getRestoreTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
    if (!restoreInstant.isPresent()) {
        throw new HoodieRollbackException("No pending restore instants found to execute restore");
    }
    try {
        List<HoodieInstant> instantsToRollback = getInstantsToRollback(restoreInstant.get());
        ValidationUtils.checkArgument(restoreInstant.get().getState().equals(HoodieInstant.State.REQUESTED) || restoreInstant.get().getState().equals(HoodieInstant.State.INFLIGHT));
        Map<String, List<HoodieRollbackMetadata>> instantToMetadata = new HashMap<>();
        if (restoreInstant.get().isRequested()) {
            table.getActiveTimeline().transitionRestoreRequestedToInflight(restoreInstant.get());
        }
        instantsToRollback.forEach(instant -> {
            instantToMetadata.put(instant.getTimestamp(), Collections.singletonList(rollbackInstant(instant)));
            LOG.info("Deleted instant " + instant);
        });
        return finishRestore(instantToMetadata, instantsToRollback, restoreTimer.endTimer());
    } catch (IOException io) {
        throw new HoodieRestoreException("unable to Restore instant " + restoreInstant.get(), io);
    }
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieTable(org.apache.hudi.table.HoodieTable) BaseActionExecutor(org.apache.hudi.table.action.BaseActionExecutor) HoodieRestorePlan(org.apache.hudi.avro.model.HoodieRestorePlan) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) Option(org.apache.hudi.common.util.Option) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) TransactionManager(org.apache.hudi.client.transaction.TransactionManager) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) HoodieInstantInfo(org.apache.hudi.avro.model.HoodieInstantInfo) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) List(java.util.List) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) LogManager(org.apache.log4j.LogManager) Collections(java.util.Collections) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HashMap(java.util.HashMap) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) ArrayList(java.util.ArrayList) List(java.util.List) IOException(java.io.IOException) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException)

Example 2 with HoodieRollbackException

use of org.apache.hudi.exception.HoodieRollbackException in project hudi by apache.

the class CopyOnWriteRestoreActionExecutor method rollbackInstant.

@Override
protected HoodieRollbackMetadata rollbackInstant(HoodieInstant instantToRollback) {
    if (!instantToRollback.getAction().equals(HoodieTimeline.COMMIT_ACTION) && !instantToRollback.getAction().equals(HoodieTimeline.REPLACE_COMMIT_ACTION)) {
        throw new HoodieRollbackException("Unsupported action in rollback instant:" + instantToRollback);
    }
    table.getMetaClient().reloadActiveTimeline();
    String newInstantTime = HoodieActiveTimeline.createNewInstantTime();
    table.scheduleRollback(context, newInstantTime, instantToRollback, false, false);
    table.getMetaClient().reloadActiveTimeline();
    CopyOnWriteRollbackActionExecutor rollbackActionExecutor = new CopyOnWriteRollbackActionExecutor(context, config, table, newInstantTime, instantToRollback, true, true, false, false);
    return rollbackActionExecutor.execute();
}
Also used : HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) CopyOnWriteRollbackActionExecutor(org.apache.hudi.table.action.rollback.CopyOnWriteRollbackActionExecutor)

Example 3 with HoodieRollbackException

use of org.apache.hudi.exception.HoodieRollbackException in project hudi by apache.

the class BaseRollbackActionExecutor method execute.

@Override
public HoodieRollbackMetadata execute() {
    table.getMetaClient().reloadActiveTimeline();
    Option<HoodieInstant> rollbackInstant = table.getRollbackTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
    if (!rollbackInstant.isPresent()) {
        throw new HoodieRollbackException("No pending rollback instants found to execute rollback");
    }
    try {
        HoodieRollbackPlan rollbackPlan = RollbackUtils.getRollbackPlan(table.getMetaClient(), rollbackInstant.get());
        return runRollback(table, rollbackInstant.get(), rollbackPlan);
    } catch (IOException e) {
        throw new HoodieIOException("Failed to fetch rollback plan for commit " + instantTime, e);
    }
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieTable(org.apache.hudi.table.HoodieTable) BaseActionExecutor(org.apache.hudi.table.action.BaseActionExecutor) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) Option(org.apache.hudi.common.util.Option) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) TransactionManager(org.apache.hudi.client.transaction.TransactionManager) HoodieTimer(org.apache.hudi.common.util.HoodieTimer) Logger(org.apache.log4j.Logger) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) BootstrapIndex(org.apache.hudi.common.bootstrap.index.BootstrapIndex) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) WriteMarkersFactory(org.apache.hudi.table.marker.WriteMarkersFactory) HoodieHeartbeatClient(org.apache.hudi.client.heartbeat.HoodieHeartbeatClient) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) IOException(java.io.IOException) HoodieRollbackPlan(org.apache.hudi.avro.model.HoodieRollbackPlan) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) List(java.util.List) ClusteringUtils(org.apache.hudi.common.util.ClusteringUtils) HoodieIOException(org.apache.hudi.exception.HoodieIOException) LogManager(org.apache.log4j.LogManager) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) Collections(java.util.Collections) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieRollbackPlan(org.apache.hudi.avro.model.HoodieRollbackPlan) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException)

Example 4 with HoodieRollbackException

use of org.apache.hudi.exception.HoodieRollbackException in project hudi by apache.

the class BaseRollbackActionExecutor method validateRollbackCommitSequence.

private void validateRollbackCommitSequence() {
    // writer mode.
    if (config.getFailedWritesCleanPolicy().isEager()) {
        final String instantTimeToRollback = instantToRollback.getTimestamp();
        HoodieTimeline commitTimeline = table.getCompletedCommitsTimeline();
        HoodieTimeline inflightAndRequestedCommitTimeline = table.getPendingCommitTimeline();
        // If there is a commit in-between or after that is not rolled back, then abort
        if ((instantTimeToRollback != null) && !commitTimeline.empty() && !commitTimeline.findInstantsAfter(instantTimeToRollback, Integer.MAX_VALUE).empty()) {
            // check if remnants are from a previous LAZY rollback config, if yes, let out of order rollback continue
            try {
                if (!HoodieHeartbeatClient.heartbeatExists(table.getMetaClient().getFs(), config.getBasePath(), instantTimeToRollback)) {
                    throw new HoodieRollbackException("Found commits after time :" + instantTimeToRollback + ", please rollback greater commits first");
                }
            } catch (IOException io) {
                throw new HoodieRollbackException("Unable to rollback commits ", io);
            }
        }
        List<String> inflights = inflightAndRequestedCommitTimeline.getInstants().map(HoodieInstant::getTimestamp).collect(Collectors.toList());
        if ((instantTimeToRollback != null) && !inflights.isEmpty() && (inflights.indexOf(instantTimeToRollback) != inflights.size() - 1)) {
            throw new HoodieRollbackException("Found in-flight commits after time :" + instantTimeToRollback + ", please rollback greater commits first");
        }
    }
}
Also used : HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException)

Example 5 with HoodieRollbackException

use of org.apache.hudi.exception.HoodieRollbackException in project hudi by apache.

the class BaseRollbackHelper method maybeDeleteAndCollectStats.

/**
 * May be delete interested files and collect stats or collect stats only.
 *
 * @param context           instance of {@link HoodieEngineContext} to use.
 * @param instantToRollback {@link HoodieInstant} of interest for which deletion or collect stats is requested.
 * @param rollbackRequests  List of {@link ListingBasedRollbackRequest} to be operated on.
 * @param doDelete          {@code true} if deletion has to be done. {@code false} if only stats are to be collected w/o performing any deletes.
 * @return stats collected with or w/o actual deletions.
 */
List<Pair<String, HoodieRollbackStat>> maybeDeleteAndCollectStats(HoodieEngineContext context, HoodieInstant instantToRollback, List<SerializableHoodieRollbackRequest> rollbackRequests, boolean doDelete, int numPartitions) {
    return context.flatMap(rollbackRequests, (SerializableFunction<SerializableHoodieRollbackRequest, Stream<Pair<String, HoodieRollbackStat>>>) rollbackRequest -> {
        List<String> filesToBeDeleted = rollbackRequest.getFilesToBeDeleted();
        if (!filesToBeDeleted.isEmpty()) {
            List<HoodieRollbackStat> rollbackStats = deleteFiles(metaClient, filesToBeDeleted, doDelete);
            List<Pair<String, HoodieRollbackStat>> partitionToRollbackStats = new ArrayList<>();
            rollbackStats.forEach(entry -> partitionToRollbackStats.add(Pair.of(entry.getPartitionPath(), entry)));
            return partitionToRollbackStats.stream();
        } else if (!rollbackRequest.getLogBlocksToBeDeleted().isEmpty()) {
            HoodieLogFormat.Writer writer = null;
            try {
                String fileId = rollbackRequest.getFileId();
                String latestBaseInstant = rollbackRequest.getLatestBaseInstant();
                writer = HoodieLogFormat.newWriterBuilder().onParentPath(FSUtils.getPartitionPath(metaClient.getBasePath(), rollbackRequest.getPartitionPath())).withFileId(fileId).overBaseCommit(latestBaseInstant).withFs(metaClient.getFs()).withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
                if (doDelete) {
                    Map<HoodieLogBlock.HeaderMetadataType, String> header = generateHeader(instantToRollback.getTimestamp());
                    writer.appendBlock(new HoodieCommandBlock(header));
                }
            } catch (IOException | InterruptedException io) {
                throw new HoodieRollbackException("Failed to rollback for instant " + instantToRollback, io);
            } finally {
                try {
                    if (writer != null) {
                        writer.close();
                    }
                } catch (IOException io) {
                    throw new HoodieIOException("Error appending rollback block", io);
                }
            }
            Map<FileStatus, Long> filesToNumBlocksRollback = Collections.singletonMap(metaClient.getFs().getFileStatus(Objects.requireNonNull(writer).getLogFile().getPath()), 1L);
            return Collections.singletonList(Pair.of(rollbackRequest.getPartitionPath(), HoodieRollbackStat.newBuilder().withPartitionPath(rollbackRequest.getPartitionPath()).withRollbackBlockAppendResults(filesToNumBlocksRollback).build())).stream();
        } else {
            return Collections.singletonList(Pair.of(rollbackRequest.getPartitionPath(), HoodieRollbackStat.newBuilder().withPartitionPath(rollbackRequest.getPartitionPath()).build())).stream();
        }
    }, numPartitions);
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) PathFilter(org.apache.hadoop.fs.PathFilter) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) HoodieCommandBlock(org.apache.hudi.common.table.log.block.HoodieCommandBlock) FileStatus(org.apache.hadoop.fs.FileStatus) ArrayList(java.util.ArrayList) Logger(org.apache.log4j.Logger) HoodieTableMetaClient(org.apache.hudi.common.table.HoodieTableMetaClient) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) HoodieLogFile(org.apache.hudi.common.model.HoodieLogFile) HoodieLogFormat(org.apache.hudi.common.table.log.HoodieLogFormat) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) SerializableFunction(org.apache.hudi.common.function.SerializableFunction) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) HoodieRollbackRequest(org.apache.hudi.avro.model.HoodieRollbackRequest) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) Serializable(java.io.Serializable) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) HoodieIOException(org.apache.hudi.exception.HoodieIOException) LogManager(org.apache.log4j.LogManager) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) HoodieLogBlock(org.apache.hudi.common.table.log.block.HoodieLogBlock) Collections(java.util.Collections) FSUtils(org.apache.hudi.common.fs.FSUtils) Pair(org.apache.hudi.common.util.collection.Pair) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) FileStatus(org.apache.hadoop.fs.FileStatus) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieCommandBlock(org.apache.hudi.common.table.log.block.HoodieCommandBlock) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieLogFormat(org.apache.hudi.common.table.log.HoodieLogFormat) Stream(java.util.stream.Stream) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

HoodieRollbackException (org.apache.hudi.exception.HoodieRollbackException)11 IOException (java.io.IOException)7 HoodieInstant (org.apache.hudi.common.table.timeline.HoodieInstant)5 HoodieWriteConfig (org.apache.hudi.config.HoodieWriteConfig)5 Collections (java.util.Collections)4 List (java.util.List)4 Collectors (java.util.stream.Collectors)4 HoodieEngineContext (org.apache.hudi.common.engine.HoodieEngineContext)4 HashMap (java.util.HashMap)3 Map (java.util.Map)3 Path (org.apache.hadoop.fs.Path)3 HoodieRollbackMetadata (org.apache.hudi.avro.model.HoodieRollbackMetadata)3 HoodieRollbackRequest (org.apache.hudi.avro.model.HoodieRollbackRequest)3 TransactionManager (org.apache.hudi.client.transaction.TransactionManager)3 HoodieRollbackStat (org.apache.hudi.common.HoodieRollbackStat)3 HoodieTimeline (org.apache.hudi.common.table.timeline.HoodieTimeline)3 HoodieIOException (org.apache.hudi.exception.HoodieIOException)3 LogManager (org.apache.log4j.LogManager)3 Logger (org.apache.log4j.Logger)3 ArrayList (java.util.ArrayList)2