Examples with HoodieRollbackRequest org.apache.hudi.avro.model.HoodieRollbackRequest used on opensource projects

Example 1 with HoodieRollbackRequest

use of org.apache.hudi.avro.model.HoodieRollbackRequest in project hudi by apache.

the class TestMarkerBasedRollbackStrategy method testCopyOnWriteRollbackWithTestTable.

@Test
public void testCopyOnWriteRollbackWithTestTable() throws Exception {
    // given: wrote some base files and corresponding markers
    HoodieTestTable testTable = HoodieTestTable.of(metaClient);
    String f0 = testTable.addRequestedCommit("000").getFileIdsWithBaseFilesInPartitions("partA").get("partA");
    String f1 = testTable.addCommit("001").withBaseFilesInPartition("partA", f0).getFileIdsWithBaseFilesInPartitions("partB").get("partB");
    String f2 = "f2";
    testTable.forCommit("001").withMarkerFile("partA", f0, IOType.MERGE).withMarkerFile("partB", f1, IOType.CREATE).withMarkerFile("partA", f2, IOType.CREATE);
    // when
    HoodieTable hoodieTable = HoodieSparkTable.create(getConfig(), context, metaClient);
    List<HoodieRollbackRequest> rollbackRequests = new MarkerBasedRollbackStrategy(hoodieTable, context, getConfig(), "002").getRollbackRequests(new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "001"));
    List<HoodieRollbackStat> stats = new BaseRollbackHelper(hoodieTable.getMetaClient(), getConfig()).performRollback(context, new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.COMMIT_ACTION, "001"), rollbackRequests);
    // then: ensure files are deleted correctly, non-existent files reported as failed deletes
    assertEquals(2, stats.size());
    FileStatus[] partAFiles = testTable.listAllFilesInPartition("partA");
    FileStatus[] partBFiles = testTable.listAllFilesInPartition("partB");
    assertEquals(0, partBFiles.length);
    assertEquals(1, partAFiles.length);
    assertEquals(2, stats.stream().mapToInt(r -> r.getSuccessDeleteFiles().size()).sum());
    assertEquals(1, stats.stream().mapToInt(r -> r.getFailedDeleteFiles().size()).sum());
}

Example 2 with HoodieRollbackRequest

use of org.apache.hudi.avro.model.HoodieRollbackRequest in project hudi by apache.

the class ListingBasedRollbackStrategy method getRollbackRequests.

@Override
public List<HoodieRollbackRequest> getRollbackRequests(HoodieInstant instantToRollback) {
    try {
        List<ListingBasedRollbackRequest> rollbackRequests = null;
        if (table.getMetaClient().getTableType() == HoodieTableType.COPY_ON_WRITE) {
            rollbackRequests = RollbackUtils.generateRollbackRequestsByListingCOW(context, table.getMetaClient().getBasePath());
        } else {
            rollbackRequests = RollbackUtils.generateRollbackRequestsUsingFileListingMOR(instantToRollback, table, context);
        }
        List<HoodieRollbackRequest> listingBasedRollbackRequests = new ListingBasedRollbackHelper(table.getMetaClient(), config).getRollbackRequestsForRollbackPlan(context, instantToRollback, rollbackRequests);
        return listingBasedRollbackRequests;
    } catch (IOException e) {
        LOG.error("Generating rollback requests failed for " + instantToRollback.getTimestamp(), e);
        throw new HoodieRollbackException("Generating rollback requests failed for " + instantToRollback.getTimestamp(), e);
    }
}

Example 3 with HoodieRollbackRequest

use of org.apache.hudi.avro.model.HoodieRollbackRequest in project hudi by apache.

the class MarkerBasedRollbackStrategy method getRollbackRequestForAppend.

protected HoodieRollbackRequest getRollbackRequestForAppend(String markerFilePath) throws IOException {
    Path baseFilePathForAppend = new Path(basePath, markerFilePath);
    String fileId = FSUtils.getFileIdFromFilePath(baseFilePathForAppend);
    String baseCommitTime = FSUtils.getCommitTime(baseFilePathForAppend.getName());
    String relativePartitionPath = FSUtils.getRelativePartitionPath(new Path(basePath), baseFilePathForAppend.getParent());
    Path partitionPath = FSUtils.getPartitionPath(config.getBasePath(), relativePartitionPath);
    // NOTE: Since we're rolling back incomplete Delta Commit, it only could have appended its
    // block to the latest log-file
    // TODO(HUDI-1517) use provided marker-file's path instead
    Option<HoodieLogFile> latestLogFileOption = FSUtils.getLatestLogFile(table.getMetaClient().getFs(), partitionPath, fileId, HoodieFileFormat.HOODIE_LOG.getFileExtension(), baseCommitTime);
    Map<String, Long> logFilesWithBlocsToRollback = new HashMap<>();
    if (latestLogFileOption.isPresent()) {
        HoodieLogFile latestLogFile = latestLogFileOption.get();
        // NOTE: Marker's don't carry information about the cumulative size of the blocks that have been appended,
        // therefore we simply stub this value.
        logFilesWithBlocsToRollback = Collections.singletonMap(latestLogFile.getFileStatus().getPath().toString(), -1L);
    }
    return new HoodieRollbackRequest(relativePartitionPath, fileId, baseCommitTime, Collections.emptyList(), logFilesWithBlocsToRollback);
}

Example 4 with HoodieRollbackRequest

use of org.apache.hudi.avro.model.HoodieRollbackRequest in project hudi by apache.

the class MarkerBasedRollbackStrategy method getRollbackRequests.

@Override
public List<HoodieRollbackRequest> getRollbackRequests(HoodieInstant instantToRollback) {
    try {
        List<String> markerPaths = MarkerBasedRollbackUtils.getAllMarkerPaths(table, context, instantToRollback.getTimestamp(), config.getRollbackParallelism());
        int parallelism = Math.max(Math.min(markerPaths.size(), config.getRollbackParallelism()), 1);
        return context.map(markerPaths, markerFilePath -> {
            String typeStr = markerFilePath.substring(markerFilePath.lastIndexOf(".") + 1);
            IOType type = IOType.valueOf(typeStr);
            switch(type) {
                case MERGE:
                case CREATE:
                    String fileToDelete = WriteMarkers.stripMarkerSuffix(markerFilePath);
                    Path fullDeletePath = new Path(basePath, fileToDelete);
                    String partitionPath = FSUtils.getRelativePartitionPath(new Path(basePath), fullDeletePath.getParent());
                    return new HoodieRollbackRequest(partitionPath, EMPTY_STRING, EMPTY_STRING, Collections.singletonList(fullDeletePath.toString()), Collections.emptyMap());
                case APPEND:
                    // - Partition path
                    return getRollbackRequestForAppend(WriteMarkers.stripMarkerSuffix(markerFilePath));
                default:
                    throw new HoodieRollbackException("Unknown marker type, during rollback of " + instantToRollback);
            }
        }, parallelism);
    } catch (Exception e) {
        throw new HoodieRollbackException("Error rolling back using marker files written for " + instantToRollback, e);
    }
}

Example 5 with HoodieRollbackRequest

use of org.apache.hudi.avro.model.HoodieRollbackRequest in project hudi by apache.

the class BaseRollbackPlanActionExecutor method requestRollback.

/**
 * Creates a Rollback plan if there are files to be rolledback and stores them in instant file.
 * Rollback Plan contains absolute file paths.
 *
 * @param startRollbackTime Rollback Instant Time
 * @return Rollback Plan if generated
 */
protected Option<HoodieRollbackPlan> requestRollback(String startRollbackTime) {
    final HoodieInstant rollbackInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, startRollbackTime);
    try {
        List<HoodieRollbackRequest> rollbackRequests = new ArrayList<>();
        if (!instantToRollback.isRequested()) {
            rollbackRequests.addAll(getRollbackStrategy().getRollbackRequests(instantToRollback));
        }
        HoodieRollbackPlan rollbackPlan = new HoodieRollbackPlan(new HoodieInstantInfo(instantToRollback.getTimestamp(), instantToRollback.getAction()), rollbackRequests, LATEST_ROLLBACK_PLAN_VERSION);
        if (!skipTimelinePublish) {
            if (table.getRollbackTimeline().filterInflightsAndRequested().containsInstant(rollbackInstant.getTimestamp())) {
                LOG.warn("Request Rollback found with instant time " + rollbackInstant + ", hence skipping scheduling rollback");
            } else {
                table.getActiveTimeline().saveToRollbackRequested(rollbackInstant, TimelineMetadataUtils.serializeRollbackPlan(rollbackPlan));
                table.getMetaClient().reloadActiveTimeline();
                LOG.info("Requesting Rollback with instant time " + rollbackInstant);
            }
        }
        return Option.of(rollbackPlan);
    } catch (IOException e) {
        LOG.error("Got exception when saving rollback requested file", e);
        throw new HoodieIOException(e.getMessage(), e);
    }
}