Examples with HoodieTimeline org.apache.hudi.common.table.timeline.HoodieTimeline used on opensource projects

Example 26 with HoodieTimeline

use of org.apache.hudi.common.table.timeline.HoodieTimeline in project hudi by apache.

the class TestTimelineUtils method testGetPartitions.

@Test
public void testGetPartitions() throws IOException {
    HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
    HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
    assertTrue(activeCommitTimeline.empty());
    // older partitions that is modified by all cleans
    String olderPartition = "0";
    for (int i = 1; i <= 5; i++) {
        String ts = i + "";
        HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, ts);
        activeTimeline.createNewInstant(instant);
        activeTimeline.saveAsComplete(instant, Option.of(getCommitMetadata(basePath, ts, ts, 2, Collections.emptyMap())));
        HoodieInstant cleanInstant = new HoodieInstant(true, HoodieTimeline.CLEAN_ACTION, ts);
        activeTimeline.createNewInstant(cleanInstant);
        activeTimeline.saveAsComplete(cleanInstant, getCleanMetadata(olderPartition, ts));
    }
    metaClient.reloadActiveTimeline();
    // verify modified partitions included cleaned data
    List<String> partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertEquals(5, partitions.size());
    assertEquals(partitions, Arrays.asList(new String[] { "0", "2", "3", "4", "5" }));
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsInRange("1", "4"));
    assertEquals(4, partitions.size());
    assertEquals(partitions, Arrays.asList(new String[] { "0", "2", "3", "4" }));
    // verify only commit actions
    partitions = TimelineUtils.getPartitionsWritten(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertEquals(4, partitions.size());
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4", "5" }));
    partitions = TimelineUtils.getPartitionsWritten(metaClient.getActiveTimeline().findInstantsInRange("1", "4"));
    assertEquals(3, partitions.size());
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4" }));
}

Example 27 with HoodieTimeline

use of org.apache.hudi.common.table.timeline.HoodieTimeline in project hudi by apache.

the class TestTimelineUtils method testGetPartitionsUnpartitioned.

@Test
public void testGetPartitionsUnpartitioned() throws IOException {
    HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
    HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
    assertTrue(activeCommitTimeline.empty());
    String partitionPath = "";
    for (int i = 1; i <= 5; i++) {
        String ts = i + "";
        HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, ts);
        activeTimeline.createNewInstant(instant);
        activeTimeline.saveAsComplete(instant, Option.of(getCommitMetadata(basePath, partitionPath, ts, 2, Collections.emptyMap())));
        HoodieInstant cleanInstant = new HoodieInstant(true, HoodieTimeline.CLEAN_ACTION, ts);
        activeTimeline.createNewInstant(cleanInstant);
        activeTimeline.saveAsComplete(cleanInstant, getCleanMetadata(partitionPath, ts));
    }
    metaClient.reloadActiveTimeline();
    // verify modified partitions included cleaned data
    List<String> partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertTrue(partitions.isEmpty());
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsInRange("1", "4"));
    assertTrue(partitions.isEmpty());
}

Example 28 with HoodieTimeline

use of org.apache.hudi.common.table.timeline.HoodieTimeline in project hudi by apache.

the class TestTimelineUtils method testGetPartitionsWithReplaceCommits.

@Test
public void testGetPartitionsWithReplaceCommits() throws IOException {
    HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
    HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
    assertTrue(activeCommitTimeline.empty());
    String ts1 = "1";
    String replacePartition = "2021/01/01";
    String newFilePartition = "2021/01/02";
    HoodieInstant instant1 = new HoodieInstant(true, HoodieTimeline.REPLACE_COMMIT_ACTION, ts1);
    activeTimeline.createNewInstant(instant1);
    // create replace metadata only with replaced file Ids (no new files created)
    activeTimeline.saveAsComplete(instant1, Option.of(getReplaceCommitMetadata(basePath, ts1, replacePartition, 2, newFilePartition, 0, Collections.emptyMap(), WriteOperationType.CLUSTER)));
    metaClient.reloadActiveTimeline();
    List<String> partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("0", 10));
    assertEquals(1, partitions.size());
    assertEquals(replacePartition, partitions.get(0));
    String ts2 = "2";
    HoodieInstant instant2 = new HoodieInstant(true, HoodieTimeline.REPLACE_COMMIT_ACTION, ts2);
    activeTimeline.createNewInstant(instant2);
    // create replace metadata only with replaced file Ids (no new files created)
    activeTimeline.saveAsComplete(instant2, Option.of(getReplaceCommitMetadata(basePath, ts2, replacePartition, 0, newFilePartition, 3, Collections.emptyMap(), WriteOperationType.CLUSTER)));
    metaClient.reloadActiveTimeline();
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertEquals(1, partitions.size());
    assertEquals(newFilePartition, partitions.get(0));
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("0", 10));
    assertEquals(2, partitions.size());
    assertTrue(partitions.contains(replacePartition));
    assertTrue(partitions.contains(newFilePartition));
}

Example 29 with HoodieTimeline

use of org.apache.hudi.common.table.timeline.HoodieTimeline in project hudi by apache.

the class TestTimelineUtils method testRestoreInstants.

@Test
public void testRestoreInstants() throws Exception {
    HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
    HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
    assertTrue(activeCommitTimeline.empty());
    for (int i = 1; i <= 5; i++) {
        String ts = i + "";
        HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, ts);
        activeTimeline.createNewInstant(instant);
        activeTimeline.saveAsComplete(instant, Option.of(getRestoreMetadata(basePath, ts, ts, 2, HoodieTimeline.COMMIT_ACTION)));
    }
    metaClient.reloadActiveTimeline();
    // verify modified partitions included cleaned data
    List<String> partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsAfter("1", 10));
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4", "5" }));
    partitions = TimelineUtils.getAffectedPartitions(metaClient.getActiveTimeline().findInstantsInRange("1", "4"));
    assertEquals(partitions, Arrays.asList(new String[] { "2", "3", "4" }));
}

Example 30 with HoodieTimeline

use of org.apache.hudi.common.table.timeline.HoodieTimeline in project hudi by apache.

the class TestCompactionUtils method testGetDeltaCommitsSinceLatestCompaction.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testGetDeltaCommitsSinceLatestCompaction(boolean hasCompletedCompaction) {
    HoodieActiveTimeline timeline = prepareTimeline(hasCompletedCompaction);
    Pair<HoodieTimeline, HoodieInstant> actual = CompactionUtils.getDeltaCommitsSinceLatestCompaction(timeline).get();
    if (hasCompletedCompaction) {
        Stream<HoodieInstant> instants = actual.getLeft().getInstants();
        assertEquals(Stream.of(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "07"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "08"), new HoodieInstant(true, HoodieTimeline.DELTA_COMMIT_ACTION, "09")).collect(Collectors.toList()), actual.getLeft().getInstants().collect(Collectors.toList()));
        assertEquals(new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, "06"), actual.getRight());
    } else {
        assertEquals(Stream.of(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "01"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "02"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "03"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "04"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "05"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "07"), new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "08"), new HoodieInstant(true, HoodieTimeline.DELTA_COMMIT_ACTION, "09")).collect(Collectors.toList()), actual.getLeft().getInstants().collect(Collectors.toList()));
        assertEquals(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "01"), actual.getRight());
    }
}