Examples with HoodieActiveTimeline org.apache.hudi.common.table.timeline.HoodieActiveTimeline used on opensource projects

Example 11 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestCompactionCommand method generateCompactionInstances.

private void generateCompactionInstances() throws IOException {
    // create MOR table.
    new TableCommand().createTable(tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(), "", TimelineLayoutVersion.VERSION_1, HoodieAvroPayload.class.getName());
    CompactionTestUtils.setupAndValidateCompactionOperations(HoodieCLI.getTableMetaClient(), true, 1, 2, 3, 4);
    HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().reloadActiveTimeline();
    // Create six commits
    Arrays.asList("001", "003", "005", "007").forEach(timestamp -> {
        activeTimeline.transitionCompactionInflightToComplete(new HoodieInstant(HoodieInstant.State.INFLIGHT, COMPACTION_ACTION, timestamp), Option.empty());
    });
    // Simulate a compaction commit in metadata table timeline
    // so the archival in data table can happen
    HoodieTestUtils.createCompactionCommitInMetadataTable(hadoopConf(), new HoodieWrapperFileSystem(FSUtils.getFs(tablePath, hadoopConf()), new NoOpConsistencyGuard()), tablePath, "007");
}

Example 12 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestRollbacksCommand method testShowRollback.

/**
 * Test case for command 'show rollback'.
 */
@Test
public void testShowRollback() throws IOException {
    // get instant
    HoodieActiveTimeline activeTimeline = new RollbacksCommand.RollbackTimeline(HoodieCLI.getTableMetaClient());
    Stream<HoodieInstant> rollback = activeTimeline.getRollbackTimeline().filterCompletedInstants().getInstants();
    HoodieInstant instant = rollback.findFirst().orElse(null);
    assertNotNull(instant, "The instant can not be null.");
    CommandResult cr = shell().executeCommand("show rollback --instant " + instant.getTimestamp());
    assertTrue(cr.isSuccess());
    List<Comparable[]> rows = new ArrayList<>();
    // get metadata of instant
    HoodieRollbackMetadata metadata = TimelineMetadataUtils.deserializeAvroMetadata(activeTimeline.getInstantDetails(instant).get(), HoodieRollbackMetadata.class);
    // generate expect result
    metadata.getPartitionMetadata().forEach((key, value) -> Stream.concat(value.getSuccessDeleteFiles().stream().map(f -> Pair.of(f, true)), value.getFailedDeleteFiles().stream().map(f -> Pair.of(f, false))).forEach(fileWithDeleteStatus -> {
        Comparable[] row = new Comparable[5];
        row[0] = metadata.getStartRollbackTime();
        row[1] = metadata.getCommitsRollback().toString();
        row[2] = key;
        row[3] = fileWithDeleteStatus.getLeft();
        row[4] = fileWithDeleteStatus.getRight();
        rows.add(row);
    }));
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_ROLLBACK_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_PARTITION).addTableHeaderField(HoodieTableHeaderFields.HEADER_DELETED_FILE).addTableHeaderField(HoodieTableHeaderFields.HEADER_SUCCEEDED);
    String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
}

Example 13 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestRollbacksCommand method testShowRollbacks.

/**
 * Test case for command 'show rollbacks'.
 */
@Test
public void testShowRollbacks() {
    CommandResult cr = shell().executeCommand("show rollbacks");
    assertTrue(cr.isSuccess());
    // get rollback instants
    HoodieActiveTimeline activeTimeline = new RollbacksCommand.RollbackTimeline(HoodieCLI.getTableMetaClient());
    Stream<HoodieInstant> rollback = activeTimeline.getRollbackTimeline().filterCompletedInstants().getInstants();
    List<Comparable[]> rows = new ArrayList<>();
    rollback.sorted().forEach(instant -> {
        try {
            // get pair of rollback time and instant time
            HoodieRollbackMetadata metadata = TimelineMetadataUtils.deserializeAvroMetadata(activeTimeline.getInstantDetails(instant).get(), HoodieRollbackMetadata.class);
            metadata.getCommitsRollback().forEach(c -> {
                Comparable[] row = new Comparable[5];
                row[0] = metadata.getStartRollbackTime();
                row[1] = c;
                // expect data
                row[2] = 3;
                row[3] = metadata.getTimeTakenInMillis();
                row[4] = 3;
                rows.add(row);
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    });
    TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_ROLLBACK_INSTANT).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_FILES_DELETED).addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS).addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_PARTITIONS);
    String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
    expected = removeNonWordAndStripSpace(expected);
    String got = removeNonWordAndStripSpace(cr.getResult().toString());
    assertEquals(expected, got);
}

Example 14 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class ITTestSavepointsCommand method testRollbackToSavepoint.

/**
 * Test case of command 'savepoint rollback'.
 */
@Test
public void testRollbackToSavepoint() throws IOException {
    // generate four savepoints
    for (int i = 100; i < 104; i++) {
        String instantTime = String.valueOf(i);
        HoodieTestDataGenerator.createCommitFile(tablePath, instantTime, jsc.hadoopConfiguration());
    }
    // generate one savepoint
    String savepoint = "102";
    HoodieTestDataGenerator.createSavepointFile(tablePath, savepoint, jsc.hadoopConfiguration());
    CommandResult cr = getShell().executeCommand(String.format("savepoint rollback --savepoint %s --sparkMaster %s", savepoint, "local"));
    assertAll("Command run failed", () -> assertTrue(cr.isSuccess()), () -> assertEquals(String.format("Savepoint \"%s\" rolled back", savepoint), cr.getResult().toString()));
    // there is 1 restore instant
    HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    assertEquals(1, timeline.getRestoreTimeline().countInstants());
    // 103 instant had rollback
    assertFalse(timeline.getCommitTimeline().containsInstant(new HoodieInstant(HoodieInstant.State.COMPLETED, "commit", "103")));
}

Example 15 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class ITTestSavepointsCommand method testDeleteSavepoint.

/**
 * Test case of command 'savepoint delete'.
 */
@Test
public void testDeleteSavepoint() throws IOException {
    // generate four savepoints
    for (int i = 100; i < 104; i++) {
        String instantTime = String.valueOf(i);
        HoodieTestDataGenerator.createCommitFile(tablePath, instantTime, jsc.hadoopConfiguration());
    }
    // generate two savepoint
    String savepoint1 = "100";
    String savepoint2 = "102";
    HoodieTestDataGenerator.createSavepointFile(tablePath, savepoint1, jsc.hadoopConfiguration());
    HoodieTestDataGenerator.createSavepointFile(tablePath, savepoint2, jsc.hadoopConfiguration());
    HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
    assertEquals(2, timeline.getSavePointTimeline().countInstants(), "There should 2 instants.");
    CommandResult cr = getShell().executeCommand(String.format("savepoint delete --commit %s --sparkMaster %s", savepoint1, "local"));
    assertAll("Command run failed", () -> assertTrue(cr.isSuccess()), () -> assertEquals(String.format("Savepoint \"%s\" deleted.", savepoint1), cr.getResult().toString()));
    // reload timeline
    timeline = timeline.reload();
    assertEquals(1, timeline.getSavePointTimeline().countInstants(), "There should 1 instants.");
    // after delete, 100 instant should not exist.
    assertFalse(timeline.containsInstant(new HoodieInstant(false, HoodieTimeline.SAVEPOINT_ACTION, savepoint1)));
}