Examples with ListingBasedRollbackRequest org.apache.hudi.table.action.rollback.ListingBasedRollbackRequest used on opensource projects

Example 1 with ListingBasedRollbackRequest

use of org.apache.hudi.table.action.rollback.ListingBasedRollbackRequest in project hudi by apache.

the class ZeroToOneUpgradeHandler method recreateMarkers.

/**
 * Recreate markers in new format.
 * Step1: Delete existing markers
 * Step2: Collect all rollback file info.
 * Step3: recreate markers for all interested files.
 *
 * @param commitInstantTime instant of interest for which markers need to be recreated.
 * @param table             instance of {@link HoodieTable} to use
 * @param context           instance of {@link HoodieEngineContext} to use
 * @throws HoodieRollbackException on any exception during upgrade.
 */
protected void recreateMarkers(final String commitInstantTime, HoodieTable table, HoodieEngineContext context, int parallelism) throws HoodieRollbackException {
    try {
        // fetch hoodie instant
        Option<HoodieInstant> commitInstantOpt = Option.fromJavaOptional(table.getActiveTimeline().getCommitsTimeline().getInstants().filter(instant -> HoodieActiveTimeline.EQUALS.test(instant.getTimestamp(), commitInstantTime)).findFirst());
        if (commitInstantOpt.isPresent()) {
            // delete existing markers
            WriteMarkers writeMarkers = WriteMarkersFactory.get(MarkerType.DIRECT, table, commitInstantTime);
            writeMarkers.quietDeleteMarkerDir(context, parallelism);
            // generate rollback stats
            List<ListingBasedRollbackRequest> rollbackRequests;
            if (table.getMetaClient().getTableType() == HoodieTableType.COPY_ON_WRITE) {
                rollbackRequests = RollbackUtils.generateRollbackRequestsByListingCOW(context, table.getMetaClient().getBasePath());
            } else {
                rollbackRequests = RollbackUtils.generateRollbackRequestsUsingFileListingMOR(commitInstantOpt.get(), table, context);
            }
            List<HoodieRollbackStat> rollbackStats = getListBasedRollBackStats(table.getMetaClient(), table.getConfig(), context, commitInstantOpt, rollbackRequests);
            // recreate markers adhering to marker based rollback
            for (HoodieRollbackStat rollbackStat : rollbackStats) {
                for (String path : rollbackStat.getSuccessDeleteFiles()) {
                    String dataFileName = path.substring(path.lastIndexOf("/") + 1);
                    // not feasible to differentiate MERGE from CREATE. hence creating with MERGE IOType for all base files.
                    writeMarkers.create(rollbackStat.getPartitionPath(), dataFileName, IOType.MERGE);
                }
                for (FileStatus fileStatus : rollbackStat.getCommandBlocksCount().keySet()) {
                    writeMarkers.create(rollbackStat.getPartitionPath(), getFileNameForMarkerFromLogFile(fileStatus.getPath().toString(), table), IOType.APPEND);
                }
            }
        }
    } catch (Exception e) {
        throw new HoodieRollbackException("Exception thrown while upgrading Hoodie Table from version 0 to 1", e);
    }
}