Examples with HoodieActiveTimeline org.apache.hudi.common.table.timeline.HoodieActiveTimeline used on opensource projects

Example 26 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestTimelineUtils method testRestoreInstants.

@Test
public void testRestoreInstants() throws Exception {
    HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
    HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
    assertTrue(activeCommitTimeline.empty());
    for (int i = 1; i <= 5; i++) {
        String ts = i + "";
        HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, ts);
        activeTimeline.createNewInstant(instant);
        activeTimeline.saveAsComplete(instant, Option.of(getRestoreMetadata(basePath, ts, ts, 2, HoodieTimeline.COMMIT_ACTION)));
    }
    metaClient.reloadActiveTimeline();
    // verify modified partitions included cleaned data
    List<String> partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4", "5" }));
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsInRange("1", "4"));
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4" }));
}

Example 27 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestCompactionUtils method testGetOldestInstantToKeepForCompactionWithEmptyDeltaCommits.

@Test
public void testGetOldestInstantToKeepForCompactionWithEmptyDeltaCommits() {
    HoodieActiveTimeline timeline = new MockHoodieActiveTimeline();
    assertEquals(Option.empty(), CompactionUtils.getOldestInstantToRetainForCompaction(timeline, 20));
}

Example 28 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestCompactionUtils method testGetDeltaCommitsSinceLatestCompaction.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testGetDeltaCommitsSinceLatestCompaction(boolean hasCompletedCompaction) {
    HoodieActiveTimeline timeline = prepareTimeline(hasCompletedCompaction);
    Pair<HoodieTimeline, HoodieInstant> actual = CompactionUtils.getDeltaCommitsSinceLatestCompaction(timeline).get();
    if (hasCompletedCompaction) {
        Stream<HoodieInstant> instants = actual.getLeft().getInstants();
        assertEquals(Stream.of(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "07"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "08"), new HoodieInstant(true, HoodieTimeline.DELTA_COMMIT_ACTION, "09")).collect(Collectors.toList()), actual.getLeft().getInstants().collect(Collectors.toList()));
        assertEquals(new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, "06"), actual.getRight());
    } else {
        assertEquals(Stream.of(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "01"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "02"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "03"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "04"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "05"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "07"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "08"), new HoodieInstant(true, HoodieTimeline.DELTA_COMMIT_ACTION, "09")).collect(Collectors.toList()), actual.getLeft().getInstants().collect(Collectors.toList()));
        assertEquals(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "01"), actual.getRight());
    }
}

Example 29 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestCompactionUtils method testGetDeltaCommitsSinceLatestCompactionWithEmptyDeltaCommits.

@Test
public void testGetDeltaCommitsSinceLatestCompactionWithEmptyDeltaCommits() {
    HoodieActiveTimeline timeline = new MockHoodieActiveTimeline();
    assertEquals(Option.empty(), CompactionUtils.getDeltaCommitsSinceLatestCompaction(timeline));
}

Example 30 with HoodieActiveTimeline

use of org.apache.hudi.common.table.timeline.HoodieActiveTimeline in project hudi by apache.

the class TestHoodieParquetInputFormat method testPendingCompactionWithActiveCommits.

// Verify that HoodieParquetInputFormat does not return instants after pending compaction
@Test
public void testPendingCompactionWithActiveCommits() throws IOException {
    // setup 4 sample instants in timeline
    List<HoodieInstant> instants = new ArrayList<>();
    HoodieInstant t1 = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "1");
    HoodieInstant t2 = new HoodieInstant(HoodieInstant.State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, "2");
    HoodieInstant t3 = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "3");
    HoodieInstant t4 = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "4");
    HoodieInstant t5 = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "5");
    HoodieInstant t6 = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "6");
    instants.add(t1);
    instants.add(t2);
    instants.add(t3);
    instants.add(t4);
    instants.add(t5);
    instants.add(t6);
    HoodieTableMetaClient metaClient = HoodieTestUtils.init(basePath.toString());
    HoodieActiveTimeline timeline = new HoodieActiveTimeline(metaClient);
    timeline.setInstants(instants);
    // Verify getCommitsTimelineBeforePendingCompaction does not return instants after first compaction instant
    HoodieTimeline filteredTimeline = HoodieInputFormatUtils.filterInstantsTimeline(timeline);
    assertTrue(filteredTimeline.containsInstant(t1));
    assertTrue(filteredTimeline.containsInstant(t2));
    assertFalse(filteredTimeline.containsInstant(t3));
    assertFalse(filteredTimeline.containsInstant(t4));
    assertFalse(filteredTimeline.containsInstant(t5));
    assertFalse(filteredTimeline.containsInstant(t6));
    // remove compaction instant and setup timeline again
    instants.remove(t3);
    timeline = new HoodieActiveTimeline(metaClient);
    timeline.setInstants(instants);
    filteredTimeline = HoodieInputFormatUtils.filterInstantsTimeline(timeline);
    // verify all remaining instants are returned.
    assertTrue(filteredTimeline.containsInstant(t1));
    assertTrue(filteredTimeline.containsInstant(t2));
    assertFalse(filteredTimeline.containsInstant(t3));
    assertTrue(filteredTimeline.containsInstant(t4));
    assertFalse(filteredTimeline.containsInstant(t5));
    assertFalse(filteredTimeline.containsInstant(t6));
    // remove remaining compaction instant and setup timeline again
    instants.remove(t5);
    timeline = new HoodieActiveTimeline(metaClient);
    timeline.setInstants(instants);
    filteredTimeline = HoodieInputFormatUtils.filterInstantsTimeline(timeline);
    // verify all remaining instants are returned.
    assertTrue(filteredTimeline.containsInstant(t1));
    assertTrue(filteredTimeline.containsInstant(t2));
    assertFalse(filteredTimeline.containsInstant(t3));
    assertTrue(filteredTimeline.containsInstant(t4));
    assertFalse(filteredTimeline.containsInstant(t5));
    assertTrue(filteredTimeline.containsInstant(t6));
}