Examples with CompactionUnit org.apache.flink.connector.file.table.stream.compact.CompactMessages.CompactionUnit used on opensource projects

Example 6 with CompactionUnit

use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.CompactionUnit in project flink by apache.

the class CompactCoordinatorTest method testCoordinatorCrossCheckpoints.

@Test
public void testCoordinatorCrossCheckpoints() throws Exception {
    AtomicReference<OperatorSubtaskState> state = new AtomicReference<>();
    runCoordinator(harness -> {
        harness.setup();
        harness.open();
        harness.processElement(new InputFile("p0", newFile("f0", 3)), 0);
        harness.processElement(new InputFile("p0", newFile("f1", 2)), 0);
        harness.processElement(new InputFile("p1", newFile("f2", 2)), 0);
        harness.processElement(new InputFile("p0", newFile("f3", 5)), 0);
        harness.processElement(new InputFile("p0", newFile("f4", 1)), 0);
        harness.processElement(new InputFile("p1", newFile("f5", 5)), 0);
        harness.processElement(new InputFile("p1", newFile("f6", 4)), 0);
        state.set(harness.snapshot(1, 0));
    });
    runCoordinator(harness -> {
        harness.setup();
        harness.initializeState(state.get());
        harness.open();
        harness.processElement(new InputFile("p0", newFile("f7", 3)), 0);
        harness.processElement(new InputFile("p0", newFile("f8", 2)), 0);
        state.set(harness.snapshot(2, 0));
    });
    runCoordinator(harness -> {
        harness.setup();
        harness.initializeState(state.get());
        harness.open();
        harness.processElement(new EndCheckpoint(2, 0, 1), 0);
        List<CoordinatorOutput> outputs = harness.extractOutputValues();
        Assert.assertEquals(7, outputs.size());
        List<CompactionUnit> cp1Units = new ArrayList<>();
        for (int i = 0; i < 4; i++) {
            CoordinatorOutput output = outputs.get(i);
            Assert.assertTrue(output instanceof CompactionUnit);
            cp1Units.add((CompactionUnit) output);
        }
        cp1Units.sort(Comparator.comparing(CompactionUnit::getPartition).thenComparingInt(CompactionUnit::getUnitId));
        assertUnit(cp1Units.get(0), 0, "p0", Arrays.asList("f0", "f1", "f4"));
        assertUnit(cp1Units.get(1), 1, "p0", Collections.singletonList("f3"));
        assertUnit(cp1Units.get(2), 2, "p1", Arrays.asList("f2", "f5"));
        assertUnit(cp1Units.get(3), 3, "p1", Collections.singletonList("f6"));
        assertEndCompaction(outputs.get(4), 1);
        assertUnit(outputs.get(5), 0, "p0", Arrays.asList("f7", "f8"));
        assertEndCompaction(outputs.get(6), 2);
    });
}

Example 7 with CompactionUnit

use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.CompactionUnit in project flink by apache.

the class CompactCoordinatorTest method assertUnit.

private void assertUnit(CoordinatorOutput output, int unitId, String partition, List<String> fileNames) {
    Assert.assertTrue(output instanceof CompactionUnit);
    CompactionUnit unit = (CompactionUnit) output;
    Assert.assertEquals(unitId, unit.getUnitId());
    Assert.assertEquals(partition, unit.getPartition());
    Assert.assertEquals(fileNames, unit.getPaths().stream().map(Path::getName).collect(Collectors.toList()));
}