Examples with HoodieRollbackPlan org.apache.hudi.avro.model.HoodieRollbackPlan used on opensource projects

Example 1 with HoodieRollbackPlan

use of org.apache.hudi.avro.model.HoodieRollbackPlan in project hudi by apache.

the class TestCopyOnWriteRollbackActionExecutor method testCopyOnWriteRollbackActionExecutorForFileListingAsGenerateFile.

@Test
public void testCopyOnWriteRollbackActionExecutorForFileListingAsGenerateFile() throws Exception {
    final String p1 = "2015/03/16";
    final String p2 = "2015/03/17";
    final String p3 = "2016/03/15";
    // Let's create some commit files and base files
    HoodieTestTable testTable = HoodieTestTable.of(metaClient).withPartitionMetaFiles(p1, p2, p3).addCommit("001").withBaseFilesInPartition(p1, "id11").withBaseFilesInPartition(p2, "id12").withLogFile(p1, "id11", 3).addCommit("002").withBaseFilesInPartition(p1, "id21").withBaseFilesInPartition(p2, "id22");
    HoodieWriteConfig writeConfig = getConfigBuilder().withRollbackUsingMarkers(false).build();
    HoodieTable table = this.getHoodieTable(metaClient, writeConfig);
    HoodieInstant needRollBackInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, "002");
    // execute CopyOnWriteRollbackActionExecutor with filelisting mode
    BaseRollbackPlanActionExecutor copyOnWriteRollbackPlanActionExecutor = new BaseRollbackPlanActionExecutor(context, table.getConfig(), table, "003", needRollBackInstant, false, table.getConfig().shouldRollbackUsingMarkers());
    HoodieRollbackPlan rollbackPlan = (HoodieRollbackPlan) copyOnWriteRollbackPlanActionExecutor.execute().get();
    CopyOnWriteRollbackActionExecutor copyOnWriteRollbackActionExecutor = new CopyOnWriteRollbackActionExecutor(context, table.getConfig(), table, "003", needRollBackInstant, true, false);
    List<HoodieRollbackStat> hoodieRollbackStats = copyOnWriteRollbackActionExecutor.executeRollback(rollbackPlan);
    // assert hoodieRollbackStats
    assertEquals(hoodieRollbackStats.size(), 3);
    for (HoodieRollbackStat stat : hoodieRollbackStats) {
        switch(stat.getPartitionPath()) {
            case p1:
                assertEquals(1, stat.getSuccessDeleteFiles().size());
                assertEquals(0, stat.getFailedDeleteFiles().size());
                assertEquals(Collections.EMPTY_MAP, stat.getCommandBlocksCount());
                assertEquals(testTable.forCommit("002").getBaseFilePath(p1, "id21").toString(), this.fs.getScheme() + ":" + stat.getSuccessDeleteFiles().get(0));
                break;
            case p2:
                assertEquals(1, stat.getSuccessDeleteFiles().size());
                assertEquals(0, stat.getFailedDeleteFiles().size());
                assertEquals(Collections.EMPTY_MAP, stat.getCommandBlocksCount());
                assertEquals(testTable.forCommit("002").getBaseFilePath(p2, "id22").toString(), this.fs.getScheme() + ":" + stat.getSuccessDeleteFiles().get(0));
                break;
            case p3:
                assertEquals(0, stat.getSuccessDeleteFiles().size());
                assertEquals(0, stat.getFailedDeleteFiles().size());
                assertEquals(Collections.EMPTY_MAP, stat.getCommandBlocksCount());
                break;
            default:
                fail("Unexpected partition: " + stat.getPartitionPath());
        }
    }
    assertTrue(testTable.inflightCommitExists("001"));
    assertTrue(testTable.commitExists("001"));
    assertTrue(testTable.baseFileExists(p1, "001", "id11"));
    assertTrue(testTable.baseFileExists(p2, "001", "id12"));
    assertFalse(testTable.inflightCommitExists("002"));
    assertFalse(testTable.commitExists("002"));
    assertFalse(testTable.baseFileExists(p1, "002", "id21"));
    assertFalse(testTable.baseFileExists(p2, "002", "id22"));
}

Example 2 with HoodieRollbackPlan

use of org.apache.hudi.avro.model.HoodieRollbackPlan in project hudi by apache.

the class BaseRollbackActionExecutor method execute.

@Override
public HoodieRollbackMetadata execute() {
    table.getMetaClient().reloadActiveTimeline();
    Option<HoodieInstant> rollbackInstant = table.getRollbackTimeline().filterInflightsAndRequested().filter(instant -> instant.getTimestamp().equals(instantTime)).firstInstant();
    if (!rollbackInstant.isPresent()) {
        throw new HoodieRollbackException("No pending rollback instants found to execute rollback");
    }
    try {
        HoodieRollbackPlan rollbackPlan = RollbackUtils.getRollbackPlan(table.getMetaClient(), rollbackInstant.get());
        return runRollback(table, rollbackInstant.get(), rollbackPlan);
    } catch (IOException e) {
        throw new HoodieIOException("Failed to fetch rollback plan for commit " + instantTime, e);
    }
}

Example 3 with HoodieRollbackPlan

use of org.apache.hudi.avro.model.HoodieRollbackPlan in project hudi by apache.

the class TestClientRollback method testFailedRollbackCommit.

/**
 * Test Cases for effects of rollbacking completed/inflight commits.
 */
@Test
public void testFailedRollbackCommit() throws Exception {
    // Let's create some commit files and base files
    final String p1 = "2016/05/01";
    final String p2 = "2016/05/02";
    final String p3 = "2016/05/06";
    final String commitTime1 = "20160501010101";
    final String commitTime2 = "20160502020601";
    final String commitTime3 = "20160506030611";
    Map<String, String> partitionAndFileId1 = new HashMap<String, String>() {

        {
            put(p1, "id11");
            put(p2, "id12");
            put(p3, "id13");
        }
    };
    Map<String, String> partitionAndFileId2 = new HashMap<String, String>() {

        {
            put(p1, "id21");
            put(p2, "id22");
            put(p3, "id23");
        }
    };
    Map<String, String> partitionAndFileId3 = new HashMap<String, String>() {

        {
            put(p1, "id31");
            put(p2, "id32");
            put(p3, "id33");
        }
    };
    HoodieTestTable testTable = HoodieTestTable.of(metaClient).withPartitionMetaFiles(p1, p2, p3).addCommit(commitTime1).withBaseFilesInPartitions(partitionAndFileId1).addCommit(commitTime2).withBaseFilesInPartitions(partitionAndFileId2).addInflightCommit(commitTime3).withBaseFilesInPartitions(partitionAndFileId3);
    HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withRollbackUsingMarkers(false).withCompactionConfig(HoodieCompactionConfig.newBuilder().withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY).build()).withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
    try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
        // Rollback commit3
        client.rollback(commitTime3);
        assertFalse(testTable.inflightCommitExists(commitTime3));
        assertFalse(testTable.baseFilesExist(partitionAndFileId3, commitTime3));
        assertTrue(testTable.baseFilesExist(partitionAndFileId2, commitTime2));
        metaClient.reloadActiveTimeline();
        List<HoodieInstant> rollbackInstants = metaClient.getActiveTimeline().getRollbackTimeline().getInstants().collect(Collectors.toList());
        assertEquals(rollbackInstants.size(), 1);
        HoodieInstant rollbackInstant = rollbackInstants.get(0);
        // delete rollback completed meta file and retry rollback.
        FileCreateUtils.deleteRollbackCommit(basePath, rollbackInstant.getTimestamp());
        // recreate actual commit files so that we can retry the rollback
        testTable.addInflightCommit(commitTime3).withBaseFilesInPartitions(partitionAndFileId3);
        // retry rolling back the commit again.
        client.rollback(commitTime3);
        // verify there are no extra rollback instants
        metaClient.reloadActiveTimeline();
        rollbackInstants = metaClient.getActiveTimeline().getRollbackTimeline().getInstants().collect(Collectors.toList());
        assertEquals(rollbackInstants.size(), 1);
        assertEquals(rollbackInstants.get(0), rollbackInstant);
        final String commitTime4 = "20160507040601";
        final String commitTime5 = "20160507050611";
        // add inflight compaction then rolls it back
        testTable.addInflightCompaction(commitTime4, new HoodieCommitMetadata());
        HoodieRollbackPlan rollbackPlan = new HoodieRollbackPlan();
        rollbackPlan.setRollbackRequests(Collections.emptyList());
        rollbackPlan.setInstantToRollback(new HoodieInstantInfo(commitTime4, HoodieTimeline.COMPACTION_ACTION));
        testTable.addRequestedRollback(commitTime5, rollbackPlan);
        // the compaction instants should be excluded
        metaClient.reloadActiveTimeline();
        assertEquals(0, client.getPendingRollbackInfos(metaClient).size());
        // verify there is no extra rollback instants
        client.rollback(commitTime4);
        metaClient.reloadActiveTimeline();
        rollbackInstants = metaClient.reloadActiveTimeline().getRollbackTimeline().getInstants().collect(Collectors.toList());
        assertEquals(2, rollbackInstants.size());
    }
}

Example 4 with HoodieRollbackPlan

use of org.apache.hudi.avro.model.HoodieRollbackPlan in project hudi by apache.

the class BaseRollbackPlanActionExecutor method requestRollback.

/**
 * Creates a Rollback plan if there are files to be rolledback and stores them in instant file.
 * Rollback Plan contains absolute file paths.
 *
 * @param startRollbackTime Rollback Instant Time
 * @return Rollback Plan if generated
 */
protected Option<HoodieRollbackPlan> requestRollback(String startRollbackTime) {
    final HoodieInstant rollbackInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, startRollbackTime);
    try {
        List<HoodieRollbackRequest> rollbackRequests = new ArrayList<>();
        if (!instantToRollback.isRequested()) {
            rollbackRequests.addAll(getRollbackStrategy().getRollbackRequests(instantToRollback));
        }
        HoodieRollbackPlan rollbackPlan = new HoodieRollbackPlan(new HoodieInstantInfo(instantToRollback.getTimestamp(), instantToRollback.getAction()), rollbackRequests, LATEST_ROLLBACK_PLAN_VERSION);
        if (!skipTimelinePublish) {
            if (table.getRollbackTimeline().filterInflightsAndRequested().containsInstant(rollbackInstant.getTimestamp())) {
                LOG.warn("Request Rollback found with instant time " + rollbackInstant + ", hence skipping scheduling rollback");
            } else {
                table.getActiveTimeline().saveToRollbackRequested(rollbackInstant, TimelineMetadataUtils.serializeRollbackPlan(rollbackPlan));
                table.getMetaClient().reloadActiveTimeline();
                LOG.info("Requesting Rollback with instant time " + rollbackInstant);
            }
        }
        return Option.of(rollbackPlan);
    } catch (IOException e) {
        LOG.error("Got exception when saving rollback requested file", e);
        throw new HoodieIOException(e.getMessage(), e);
    }
}

Example 5 with HoodieRollbackPlan

use of org.apache.hudi.avro.model.HoodieRollbackPlan in project hudi by apache.

the class BaseHoodieWriteClient method rollback.

/**
 * @Deprecated
 * Rollback the inflight record changes with the given commit time. This
 * will be removed in future in favor of {@link BaseHoodieWriteClient#restoreToInstant(String)}
 *
 * @param commitInstantTime Instant time of the commit
 * @param pendingRollbackInfo pending rollback instant and plan if rollback failed from previous attempt.
 * @param skipLocking if this is triggered by another parent transaction, locking can be skipped.
 * @throws HoodieRollbackException if rollback cannot be performed successfully
 */
@Deprecated
public boolean rollback(final String commitInstantTime, Option<HoodiePendingRollbackInfo> pendingRollbackInfo, boolean skipLocking) throws HoodieRollbackException {
    LOG.info("Begin rollback of instant " + commitInstantTime);
    final String rollbackInstantTime = pendingRollbackInfo.map(entry -> entry.getRollbackInstant().getTimestamp()).orElse(HoodieActiveTimeline.createNewInstantTime());
    final Timer.Context timerContext = this.metrics.getRollbackCtx();
    try {
        HoodieTable<T, I, K, O> table = createTable(config, hadoopConf);
        Option<HoodieInstant> commitInstantOpt = Option.fromJavaOptional(table.getActiveTimeline().getCommitsTimeline().getInstants().filter(instant -> HoodieActiveTimeline.EQUALS.test(instant.getTimestamp(), commitInstantTime)).findFirst());
        if (commitInstantOpt.isPresent()) {
            LOG.info("Scheduling Rollback at instant time :" + rollbackInstantTime);
            Option<HoodieRollbackPlan> rollbackPlanOption = pendingRollbackInfo.map(entry -> Option.of(entry.getRollbackPlan())).orElseGet(() -> table.scheduleRollback(context, rollbackInstantTime, commitInstantOpt.get(), false, config.shouldRollbackUsingMarkers()));
            if (rollbackPlanOption.isPresent()) {
                // execute rollback
                HoodieRollbackMetadata rollbackMetadata = table.rollback(context, rollbackInstantTime, commitInstantOpt.get(), true, skipLocking);
                if (timerContext != null) {
                    long durationInMs = metrics.getDurationInMs(timerContext.stop());
                    metrics.updateRollbackMetrics(durationInMs, rollbackMetadata.getTotalFilesDeleted());
                }
                return true;
            } else {
                throw new HoodieRollbackException("Failed to rollback " + config.getBasePath() + " commits " + commitInstantTime);
            }
        } else {
            LOG.warn("Cannot find instant " + commitInstantTime + " in the timeline, for rollback");
            return false;
        }
    } catch (Exception e) {
        throw new HoodieRollbackException("Failed to rollback " + config.getBasePath() + " commits " + commitInstantTime, e);
    }
}