Examples with HoodieRollbackStat org.apache.hudi.common.HoodieRollbackStat used on opensource projects

Example 11 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class TimelineMetadataUtils method convertRollbackMetadata.

public static HoodieRollbackMetadata convertRollbackMetadata(String startRollbackTime, Option<Long> durationInMs, List<HoodieInstant> instants, List<HoodieRollbackStat> rollbackStats) {
    Map<String, HoodieRollbackPartitionMetadata> partitionMetadataBuilder = new HashMap<>();
    int totalDeleted = 0;
    for (HoodieRollbackStat stat : rollbackStats) {
        Map<String, Long> rollbackLogFiles = stat.getCommandBlocksCount().keySet().stream().collect(Collectors.toMap(f -> f.getPath().toString(), FileStatus::getLen));
        HoodieRollbackPartitionMetadata metadata = new HoodieRollbackPartitionMetadata(stat.getPartitionPath(), stat.getSuccessDeleteFiles(), stat.getFailedDeleteFiles(), rollbackLogFiles);
        partitionMetadataBuilder.put(stat.getPartitionPath(), metadata);
        totalDeleted += stat.getSuccessDeleteFiles().size();
    }
    return new HoodieRollbackMetadata(startRollbackTime, durationInMs.orElseGet(() -> -1L), totalDeleted, instants.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()), Collections.unmodifiableMap(partitionMetadataBuilder), DEFAULT_VERSION, instants.stream().map(instant -> new HoodieInstantInfo(instant.getTimestamp(), instant.getAction())).collect(Collectors.toList()));
}

Example 12 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class TestRollbackUtils method testMergeRollbackStat.

@Test
public void testMergeRollbackStat() {
    String partitionPath1 = "/partitionPath1/";
    String partitionPath2 = "/partitionPath2/";
    // prepare HoodieRollbackStat for different partition
    Map<FileStatus, Boolean> dataFilesOnlyStat1Files = new HashMap<>();
    dataFilesOnlyStat1Files.put(generateFileStatus(partitionPath1 + "dataFile1" + BASE_FILE_EXTENSION), true);
    dataFilesOnlyStat1Files.put(generateFileStatus(partitionPath1 + "dataFile2" + BASE_FILE_EXTENSION), true);
    HoodieRollbackStat dataFilesOnlyStat1 = HoodieRollbackStat.newBuilder().withPartitionPath(partitionPath1).withDeletedFileResults(dataFilesOnlyStat1Files).build();
    Map<FileStatus, Boolean> dataFilesOnlyStat2Files = new HashMap<>();
    dataFilesOnlyStat2Files.put(generateFileStatus(partitionPath2 + "dataFile1" + BASE_FILE_EXTENSION), true);
    dataFilesOnlyStat2Files.put(generateFileStatus(partitionPath2 + "dataFile2" + BASE_FILE_EXTENSION), true);
    HoodieRollbackStat dataFilesOnlyStat2 = HoodieRollbackStat.newBuilder().withPartitionPath(partitionPath2).withDeletedFileResults(dataFilesOnlyStat1Files).build();
    // 1. test different partitionpath merge
    assertThrows(IllegalArgumentException.class, () -> {
        RollbackUtils.mergeRollbackStat(dataFilesOnlyStat1, dataFilesOnlyStat2);
    }, "different partition rollbackstat merge will failed");
    // prepare HoodieRollbackStat for failed and block append
    Map<FileStatus, Boolean> dataFilesOnlyStat3Files = new HashMap<>();
    dataFilesOnlyStat3Files.put(generateFileStatus(partitionPath1 + "dataFile1.log"), true);
    dataFilesOnlyStat3Files.put(generateFileStatus(partitionPath1 + "dataFile3" + BASE_FILE_EXTENSION), false);
    HoodieRollbackStat dataFilesOnlyStat3 = HoodieRollbackStat.newBuilder().withPartitionPath(partitionPath1).withDeletedFileResults(dataFilesOnlyStat3Files).build();
    Map<FileStatus, Long> dataFilesOnlyStat4Files = new HashMap<>();
    dataFilesOnlyStat4Files.put(generateFileStatus(partitionPath1 + "dataFile1.log"), 10L);
    HoodieRollbackStat dataFilesOnlyStat4 = HoodieRollbackStat.newBuilder().withPartitionPath(partitionPath1).withRollbackBlockAppendResults(dataFilesOnlyStat4Files).build();
    // 2. test merge dataFilesOnlyStat1 and dataFilesOnlyStat3
    HoodieRollbackStat dataFilesOnlyStatMerge1 = RollbackUtils.mergeRollbackStat(dataFilesOnlyStat1, dataFilesOnlyStat3);
    assertEquals(partitionPath1, dataFilesOnlyStatMerge1.getPartitionPath());
    assertIterableEquals(CollectionUtils.createImmutableList(partitionPath1 + "dataFile3" + BASE_FILE_EXTENSION), dataFilesOnlyStatMerge1.getFailedDeleteFiles());
    assertIterableEquals(CollectionUtils.createImmutableList(partitionPath1 + "dataFile1" + BASE_FILE_EXTENSION, partitionPath1 + "dataFile2" + BASE_FILE_EXTENSION, partitionPath1 + "dataFile1.log").stream().sorted().collect(Collectors.toList()), dataFilesOnlyStatMerge1.getSuccessDeleteFiles().stream().sorted().collect(Collectors.toList()));
    assertEquals(0, dataFilesOnlyStatMerge1.getCommandBlocksCount().size());
    // 3. test merge dataFilesOnlyStatMerge1 and dataFilesOnlyStat4
    HoodieRollbackStat dataFilesOnlyStatMerge2 = RollbackUtils.mergeRollbackStat(dataFilesOnlyStatMerge1, dataFilesOnlyStat4);
    assertEquals(partitionPath1, dataFilesOnlyStatMerge1.getPartitionPath());
    assertIterableEquals(CollectionUtils.createImmutableList(partitionPath1 + "dataFile3" + BASE_FILE_EXTENSION).stream().sorted().collect(Collectors.toList()), dataFilesOnlyStatMerge2.getFailedDeleteFiles().stream().sorted().collect(Collectors.toList()));
    assertIterableEquals(CollectionUtils.createImmutableList(partitionPath1 + "dataFile1" + BASE_FILE_EXTENSION, partitionPath1 + "dataFile2" + BASE_FILE_EXTENSION, partitionPath1 + "dataFile1.log").stream().sorted().collect(Collectors.toList()), dataFilesOnlyStatMerge2.getSuccessDeleteFiles().stream().sorted().collect(Collectors.toList()));
    assertEquals(CollectionUtils.createImmutableMap(generateFileStatus(partitionPath1 + "dataFile1.log"), 10L), dataFilesOnlyStatMerge2.getCommandBlocksCount());
}

Example 13 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class TestMarkerBasedRollbackStrategy method testMergeOnReadRollback.

@ParameterizedTest(name = TEST_NAME_WITH_PARAMS)
@MethodSource("configParams")
public void testMergeOnReadRollback(boolean useFileListingMetadata) throws Exception {
    // init MERGE_ON_READ_TABLE
    tearDown();
    tableType = HoodieTableType.MERGE_ON_READ;
    setUp();
    HoodieWriteConfig writeConfig = getConfigBuilder().withRollbackUsingMarkers(true).withAutoCommit(false).withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(useFileListingMetadata).build()).withPath(basePath).build();
    HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
    try (SparkRDDWriteClient writeClient = new SparkRDDWriteClient(engineContext, writeConfig)) {
        // rollback 2nd commit and ensure stats reflect the info.
        List<HoodieRollbackStat> stats = testRun(useFileListingMetadata, writeConfig, writeClient);
        assertEquals(3, stats.size());
        for (HoodieRollbackStat stat : stats) {
            assertEquals(0, stat.getSuccessDeleteFiles().size());
            assertEquals(0, stat.getFailedDeleteFiles().size());
            assertEquals(1, stat.getCommandBlocksCount().size());
            stat.getCommandBlocksCount().forEach((fileStatus, len) -> assertTrue(fileStatus.getPath().getName().contains(HoodieFileFormat.HOODIE_LOG.getFileExtension())));
        }
    }
}

Example 14 with HoodieRollbackStat

use of org.apache.hudi.common.HoodieRollbackStat in project hudi by apache.

the class TestIncrementalFSViewSync method performRestore.

/**
 * Simulate Restore of an instant in timeline and fsview.
 *
 * @param instant Instant to be rolled-back
 * @param files List of files to be deleted as part of rollback
 * @param rollbackInstant Restore Instant
 */
private void performRestore(HoodieInstant instant, List<String> files, String rollbackInstant, boolean isRestore) throws IOException {
    Map<String, List<String>> partititonToFiles = deleteFiles(files);
    List<HoodieRollbackStat> rollbackStats = partititonToFiles.entrySet().stream().map(e -> new HoodieRollbackStat(e.getKey(), e.getValue(), new ArrayList<>(), new HashMap<>())).collect(Collectors.toList());
    List<HoodieInstant> rollbacks = new ArrayList<>();
    rollbacks.add(instant);
    HoodieRollbackMetadata rollbackMetadata = TimelineMetadataUtils.convertRollbackMetadata(rollbackInstant, Option.empty(), rollbacks, rollbackStats);
    if (isRestore) {
        List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
        rollbackM.add(rollbackMetadata);
        HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(rollbackInstant, 100, Collections.singletonList(instant), CollectionUtils.createImmutableMap(rollbackInstant, rollbackM));
        HoodieInstant restoreInstant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, rollbackInstant);
        metaClient.getActiveTimeline().createNewInstant(restoreInstant);
        metaClient.getActiveTimeline().saveAsComplete(restoreInstant, TimelineMetadataUtils.serializeRestoreMetadata(metadata));
    } else {
        metaClient.getActiveTimeline().createNewInstant(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant));
        metaClient.getActiveTimeline().saveAsComplete(new HoodieInstant(true, HoodieTimeline.ROLLBACK_ACTION, rollbackInstant), TimelineMetadataUtils.serializeRollbackMetadata(rollbackMetadata));
    }
    boolean deleted = metaClient.getFs().delete(new Path(metaClient.getMetaPath(), instant.getFileName()), false);
    assertTrue(deleted);
}