Search in sources :

Example 16 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class BaseHoodieWriteClient method rollback.

/**
 * @Deprecated
 * Rollback the inflight record changes with the given commit time. This
 * will be removed in future in favor of {@link BaseHoodieWriteClient#restoreToInstant(String)}
 *
 * @param commitInstantTime Instant time of the commit
 * @param pendingRollbackInfo pending rollback instant and plan if rollback failed from previous attempt.
 * @param skipLocking if this is triggered by another parent transaction, locking can be skipped.
 * @throws HoodieRollbackException if rollback cannot be performed successfully
 */
@Deprecated
public boolean rollback(final String commitInstantTime, Option<HoodiePendingRollbackInfo> pendingRollbackInfo, boolean skipLocking) throws HoodieRollbackException {
    LOG.info("Begin rollback of instant " + commitInstantTime);
    final String rollbackInstantTime = pendingRollbackInfo.map(entry -> entry.getRollbackInstant().getTimestamp()).orElse(HoodieActiveTimeline.createNewInstantTime());
    final Timer.Context timerContext = this.metrics.getRollbackCtx();
    try {
        HoodieTable<T, I, K, O> table = createTable(config, hadoopConf);
        Option<HoodieInstant> commitInstantOpt = Option.fromJavaOptional(table.getActiveTimeline().getCommitsTimeline().getInstants().filter(instant -> HoodieActiveTimeline.EQUALS.test(instant.getTimestamp(), commitInstantTime)).findFirst());
        if (commitInstantOpt.isPresent()) {
            LOG.info("Scheduling Rollback at instant time :" + rollbackInstantTime);
            Option<HoodieRollbackPlan> rollbackPlanOption = pendingRollbackInfo.map(entry -> Option.of(entry.getRollbackPlan())).orElseGet(() -> table.scheduleRollback(context, rollbackInstantTime, commitInstantOpt.get(), false, config.shouldRollbackUsingMarkers()));
            if (rollbackPlanOption.isPresent()) {
                // execute rollback
                HoodieRollbackMetadata rollbackMetadata = table.rollback(context, rollbackInstantTime, commitInstantOpt.get(), true, skipLocking);
                if (timerContext != null) {
                    long durationInMs = metrics.getDurationInMs(timerContext.stop());
                    metrics.updateRollbackMetrics(durationInMs, rollbackMetadata.getTotalFilesDeleted());
                }
                return true;
            } else {
                throw new HoodieRollbackException("Failed to rollback " + config.getBasePath() + " commits " + commitInstantTime);
            }
        } else {
            LOG.warn("Cannot find instant " + commitInstantTime + " in the timeline, for rollback");
            return false;
        }
    } catch (Exception e) {
        throw new HoodieRollbackException("Failed to rollback " + config.getBasePath() + " commits " + commitInstantTime, e);
    }
}
Also used : HoodieTable(org.apache.hudi.table.HoodieTable) HoodieRestorePlan(org.apache.hudi.avro.model.HoodieRestorePlan) HoodieFailedWritesCleaningPolicy(org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieException(org.apache.hudi.exception.HoodieException) HoodiePendingRollbackInfo(org.apache.hudi.common.HoodiePendingRollbackInfo) TransactionManager(org.apache.hudi.client.transaction.TransactionManager) Logger(org.apache.log4j.Logger) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) ParseException(java.text.ParseException) HoodieWriteMetadata(org.apache.hudi.table.action.HoodieWriteMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) SupportsUpgradeDowngrade(org.apache.hudi.table.upgrade.SupportsUpgradeDowngrade) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) TableServiceType(org.apache.hudi.common.model.TableServiceType) HoodieMetrics(org.apache.hudi.metrics.HoodieMetrics) RollbackUtils(org.apache.hudi.table.action.rollback.RollbackUtils) HoodieCleanerPlan(org.apache.hudi.avro.model.HoodieCleanerPlan) Collection(java.util.Collection) HoodieClusteringPlan(org.apache.hudi.avro.model.HoodieClusteringPlan) HoodieRollbackPlan(org.apache.hudi.avro.model.HoodieRollbackPlan) Collectors(java.util.stream.Collectors) HoodieIndex(org.apache.hudi.index.HoodieIndex) StandardCharsets(java.nio.charset.StandardCharsets) List(java.util.List) SavepointHelpers(org.apache.hudi.table.action.savepoint.SavepointHelpers) Stream(java.util.stream.Stream) EmbeddedTimelineService(org.apache.hudi.client.embedded.EmbeddedTimelineService) HoodieWriteStat(org.apache.hudi.common.model.HoodieWriteStat) ClusteringUtils(org.apache.hudi.common.util.ClusteringUtils) HoodieCompactionPlan(org.apache.hudi.avro.model.HoodieCompactionPlan) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) Timer(com.codahale.metrics.Timer) WriteOperationType(org.apache.hudi.common.model.WriteOperationType) HoodieWriteCommitCallbackMessage(org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) Option(org.apache.hudi.common.util.Option) HoodieCommitException(org.apache.hudi.exception.HoodieCommitException) HashMap(java.util.HashMap) HoodieEngineContext(org.apache.hudi.common.engine.HoodieEngineContext) CommitUtils(org.apache.hudi.common.util.CommitUtils) State(org.apache.hudi.common.table.timeline.HoodieInstant.State) AsyncCleanerService(org.apache.hudi.async.AsyncCleanerService) LinkedHashMap(java.util.LinkedHashMap) CleanerUtils(org.apache.hudi.common.util.CleanerUtils) BulkInsertPartitioner(org.apache.hudi.table.BulkInsertPartitioner) HoodieTableMetaClient(org.apache.hudi.common.table.HoodieTableMetaClient) UpgradeDowngrade(org.apache.hudi.table.upgrade.UpgradeDowngrade) AsyncArchiveService(org.apache.hudi.async.AsyncArchiveService) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) WriteMarkersFactory(org.apache.hudi.table.marker.WriteMarkersFactory) HoodieSavepointException(org.apache.hudi.exception.HoodieSavepointException) HoodieCommitMetadata(org.apache.hudi.common.model.HoodieCommitMetadata) IOException(java.io.IOException) HoodieTableVersion(org.apache.hudi.common.table.HoodieTableVersion) TransactionUtils(org.apache.hudi.client.utils.TransactionUtils) HoodieCompactionConfig(org.apache.hudi.config.HoodieCompactionConfig) HoodieRecordPayload(org.apache.hudi.common.model.HoodieRecordPayload) HeartbeatUtils(org.apache.hudi.client.heartbeat.HeartbeatUtils) HoodieCleanMetadata(org.apache.hudi.avro.model.HoodieCleanMetadata) HoodieWriteCommitCallback(org.apache.hudi.callback.HoodieWriteCommitCallback) HoodieCommitCallbackFactory(org.apache.hudi.callback.util.HoodieCommitCallbackFactory) HoodieKey(org.apache.hudi.common.model.HoodieKey) HoodieTableMetadataWriter(org.apache.hudi.metadata.HoodieTableMetadataWriter) HoodieIOException(org.apache.hudi.exception.HoodieIOException) HoodieClusteringConfig(org.apache.hudi.config.HoodieClusteringConfig) LogManager(org.apache.log4j.LogManager) Collections(java.util.Collections) Pair(org.apache.hudi.common.util.collection.Pair) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieRollbackPlan(org.apache.hudi.avro.model.HoodieRollbackPlan) HoodieException(org.apache.hudi.exception.HoodieException) ParseException(java.text.ParseException) HoodieRollbackException(org.apache.hudi.exception.HoodieRollbackException) HoodieRestoreException(org.apache.hudi.exception.HoodieRestoreException) HoodieCommitException(org.apache.hudi.exception.HoodieCommitException) HoodieSavepointException(org.apache.hudi.exception.HoodieSavepointException) IOException(java.io.IOException) HoodieIOException(org.apache.hudi.exception.HoodieIOException) Timer(com.codahale.metrics.Timer)

Example 17 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class IncrementalTimelineSyncFileSystemView method addRollbackInstant.

/**
 * Add newly found rollback instant.
 *
 * @param timeline Hoodie Timeline
 * @param instant Rollback Instant
 */
private void addRollbackInstant(HoodieTimeline timeline, HoodieInstant instant) throws IOException {
    LOG.info("Syncing rollback instant (" + instant + ")");
    HoodieRollbackMetadata metadata = TimelineMetadataUtils.deserializeAvroMetadata(timeline.getInstantDetails(instant).get(), HoodieRollbackMetadata.class);
    metadata.getPartitionMetadata().entrySet().stream().forEach(e -> {
        removeFileSlicesForPartition(timeline, instant, e.getKey(), e.getValue().getSuccessDeleteFiles());
    });
    LOG.info("Done Syncing rollback instant (" + instant + ")");
}
Also used : HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata)

Example 18 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class TestCleaner method testInsertAndCleanFailedWritesByVersions.

/**
 * Test Helper for cleaning failed writes by versions logic from HoodieWriteClient API perspective.
 *
 * @param insertFn Insert API to be tested
 * @param isPreppedAPI Flag to indicate if a prepped-version is used. If true, a wrapper function will be used during
 *        record generation to also tag the regards (de-dupe is implicit as we use unique record-gen APIs)
 * @throws Exception in case of errors
 */
private void testInsertAndCleanFailedWritesByVersions(Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> insertFn, boolean isPreppedAPI) throws Exception {
    // keep upto 3 versions for each file
    int maxVersions = 3;
    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withHeartbeatIntervalInMs(3000).withCompactionConfig(HoodieCompactionConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(maxVersions).build()).withParallelism(1, 1).withBulkInsertParallelism(1).withFinalizeWriteParallelism(1).withDeleteParallelism(1).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build()).build();
    try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
        final Function2<List<HoodieRecord>, String, Integer> recordInsertGenWrappedFunction = generateWrapRecordsFn(isPreppedAPI, cfg, dataGen::generateInserts);
        Pair<String, JavaRDD<WriteStatus>> result = insertFirstBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn, HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS);
        client.commit(result.getLeft(), result.getRight());
        HoodieTable table = HoodieSparkTable.create(client.getConfig(), context, metaClient);
        assertTrue(table.getCompletedCleanTimeline().empty());
        insertFirstFailedBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn, HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS);
        insertFirstFailedBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn, HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS);
        Pair<String, JavaRDD<WriteStatus>> ret = insertFirstFailedBigBatchForClientCleanerTest(cfg, client, recordInsertGenWrappedFunction, insertFn, HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS);
        // Await till enough time passes such that the last failed commits heartbeats are expired
        await().atMost(10, TimeUnit.SECONDS).until(() -> client.getHeartbeatClient().isHeartbeatExpired(ret.getLeft()));
        List<HoodieCleanStat> cleanStats = runCleaner(cfg);
        assertEquals(0, cleanStats.size(), "Must not clean any files");
        HoodieActiveTimeline timeline = metaClient.reloadActiveTimeline();
        assertTrue(timeline.getTimelineOfActions(CollectionUtils.createSet(HoodieTimeline.ROLLBACK_ACTION)).filterCompletedInstants().countInstants() == 3);
        Option<HoodieInstant> rollBackInstantForFailedCommit = timeline.getTimelineOfActions(CollectionUtils.createSet(HoodieTimeline.ROLLBACK_ACTION)).filterCompletedInstants().lastInstant();
        HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.deserializeAvroMetadata(timeline.getInstantDetails(rollBackInstantForFailedCommit.get()).get(), HoodieRollbackMetadata.class);
        // Rollback of one of the failed writes should have deleted 3 files
        assertEquals(3, rollbackMetadata.getTotalFilesDeleted());
    }
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) SparkRDDWriteClient(org.apache.hudi.client.SparkRDDWriteClient) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) HoodieActiveTimeline(org.apache.hudi.common.table.timeline.HoodieActiveTimeline) HoodieWriteConfig(org.apache.hudi.config.HoodieWriteConfig) JavaRDD(org.apache.spark.api.java.JavaRDD) HoodieCleanStat(org.apache.hudi.common.HoodieCleanStat) ArrayList(java.util.ArrayList) List(java.util.List)

Example 19 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class TestHoodieTimelineArchiver method createRollbackMetadata.

private HoodieInstant createRollbackMetadata(String rollbackTime, String commitToRollback, boolean inflight, boolean isEmpty) throws IOException {
    if (inflight) {
        HoodieTestTable.of(metaClient).addInflightRollback(rollbackTime);
    } else {
        HoodieRollbackMetadata hoodieRollbackMetadata = HoodieRollbackMetadata.newBuilder().setVersion(1).setStartRollbackTime(rollbackTime).setTotalFilesDeleted(1).setTimeTakenInMillis(1000).setCommitsRollback(Collections.singletonList(commitToRollback)).setPartitionMetadata(Collections.emptyMap()).setInstantsRollback(Collections.emptyList()).build();
        HoodieTestTable.of(metaClient).addRollback(rollbackTime, hoodieRollbackMetadata, isEmpty);
    }
    return new HoodieInstant(inflight, "rollback", rollbackTime);
}
Also used : HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata)

Example 20 with HoodieRollbackMetadata

use of org.apache.hudi.avro.model.HoodieRollbackMetadata in project hudi by apache.

the class TestIncrementalFSViewSync method performRestore.

/**
 * Simulate Restore of an instant in timeline and fsview.
 *
 * @param instant Instant to be rolled-back
 * @param files List of files to be deleted as part of rollback
 * @param rollbackInstant Restore Instant
 */
private void performRestore(HoodieInstant instant, List<String> files, String rollbackInstant, boolean isRestore) throws IOException {
    Map<String, List<String>> partititonToFiles = deleteFiles(files);
    List<HoodieRollbackStat> rollbackStats = partititonToFiles.entrySet().stream().map(e -> new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>())).collect(Collectors.toList());
    List<HoodieInstant> rollbacks = new ArrayList<>();
    rollbacks.add(instant);
    HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.convertRollbackMetadata(rollbackInstant, Option.empty(), rollbacks, rollbackStats);
    if (isRestore) {
        List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
        rollbackM.add(rollbackMetadata);
        HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(rollbackInstant, 100, Collections.singletonList(instant), CollectionUtils.createImmutableMap(rollbackInstant, rollbackM));
        HoodieInstant restoreInstant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, rollbackInstant);
        metaClient.getActiveTimeline().createNewInstant(restoreInstant);
        metaClient.getActiveTimeline().saveAsComplete(restoreInstant, TimelineMetadataUtils.serializeRestoreMetadata(metadata));
    } else {
        metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant));
        metaClient.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant), TimelineMetadataUtils.serializeRollbackMetadata(rollbackMetadata));
    }
    boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
    assertTrue(deleted);
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) HoodieException(org.apache.hudi.exception.HoodieException) CollectionUtils(org.apache.hudi.common.util.CollectionUtils) COMPACTION_ACTION(org.apache.hudi.common.table.timeline.HoodieTimeline.COMPACTION_ACTION) Logger(org.apache.log4j.Logger) HoodieTableType(org.apache.hudi.common.model.HoodieTableType) HoodieFileGroup(org.apache.hudi.common.model.HoodieFileGroup) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) Map(java.util.Map) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) Path(org.apache.hadoop.fs.Path) HoodieFileGroupId(org.apache.hudi.common.model.HoodieFileGroupId) ValidationUtils(org.apache.hudi.common.util.ValidationUtils) Set(java.util.Set) TimelineMetadataUtils(org.apache.hudi.common.table.timeline.TimelineMetadataUtils) UUID(java.util.UUID) HoodieCommonTestHarness(org.apache.hudi.common.testutils.HoodieCommonTestHarness) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) CompactionOperation(org.apache.hudi.common.model.CompactionOperation) Test(org.junit.jupiter.api.Test) HoodieBaseFile(org.apache.hudi.common.model.HoodieBaseFile) List(java.util.List) HoodieWriteStat(org.apache.hudi.common.model.HoodieWriteStat) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) HoodieCompactionPlan(org.apache.hudi.avro.model.HoodieCompactionPlan) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) WriteOperationType(org.apache.hudi.common.model.WriteOperationType) CompactionUtils(org.apache.hudi.common.util.CompactionUtils) IntStream(java.util.stream.IntStream) HoodieCleaningPolicy(org.apache.hudi.common.model.HoodieCleaningPolicy) FileSlice(org.apache.hudi.common.model.FileSlice) Option(org.apache.hudi.common.util.Option) HashMap(java.util.HashMap) State(org.apache.hudi.common.table.timeline.HoodieInstant.State) ArrayList(java.util.ArrayList) HoodieRequestedReplaceMetadata(org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata) CleanerUtils(org.apache.hudi.common.util.CleanerUtils) HoodieTableMetaClient(org.apache.hudi.common.table.HoodieTableMetaClient) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) HoodieTimeline(org.apache.hudi.common.table.timeline.HoodieTimeline) HoodieCleanStat(org.apache.hudi.common.HoodieCleanStat) Files(java.nio.file.Files) HoodieCommitMetadata(org.apache.hudi.common.model.HoodieCommitMetadata) IOException(java.io.IOException) File(java.io.File) HoodieReplaceCommitMetadata(org.apache.hudi.common.model.HoodieReplaceCommitMetadata) HoodieCleanMetadata(org.apache.hudi.avro.model.HoodieCleanMetadata) Paths(java.nio.file.Paths) HoodieIOException(org.apache.hudi.exception.HoodieIOException) LogManager(org.apache.log4j.LogManager) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) Comparator(java.util.Comparator) Collections(java.util.Collections) FSUtils(org.apache.hudi.common.fs.FSUtils) Pair(org.apache.hudi.common.util.collection.Pair) HoodieInstant(org.apache.hudi.common.table.timeline.HoodieInstant) Path(org.apache.hadoop.fs.Path) HoodieRollbackMetadata(org.apache.hudi.avro.model.HoodieRollbackMetadata) ArrayList(java.util.ArrayList) HoodieRestoreMetadata(org.apache.hudi.avro.model.HoodieRestoreMetadata) HoodieRollbackStat(org.apache.hudi.common.HoodieRollbackStat) List(java.util.List) ArrayList(java.util.ArrayList)

Aggregations

HoodieRollbackMetadata (org.apache.hudi.avro.model.HoodieRollbackMetadata)23 HoodieInstant (org.apache.hudi.common.table.timeline.HoodieInstant)15 ArrayList (java.util.ArrayList)14 List (java.util.List)14 IOException (java.io.IOException)12 HashMap (java.util.HashMap)12 HoodieActiveTimeline (org.apache.hudi.common.table.timeline.HoodieActiveTimeline)12 Map (java.util.Map)10 HoodieRestoreMetadata (org.apache.hudi.avro.model.HoodieRestoreMetadata)8 HoodieTimeline (org.apache.hudi.common.table.timeline.HoodieTimeline)8 Collections (java.util.Collections)7 Collectors (java.util.stream.Collectors)7 Option (org.apache.hudi.common.util.Option)7 ValidationUtils (org.apache.hudi.common.util.ValidationUtils)7 HoodieWriteConfig (org.apache.hudi.config.HoodieWriteConfig)7 Logger (org.apache.log4j.Logger)7 HoodieTableMetaClient (org.apache.hudi.common.table.HoodieTableMetaClient)6 LogManager (org.apache.log4j.LogManager)6 HoodieCleanMetadata (org.apache.hudi.avro.model.HoodieCleanMetadata)5 HoodieInstantInfo (org.apache.hudi.avro.model.HoodieInstantInfo)5