use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.InputFile in project flink by apache.
the class CompactCoordinator method processElement.
@Override
public void processElement(StreamRecord<CoordinatorInput> element) throws Exception {
CoordinatorInput value = element.getValue();
if (value instanceof InputFile) {
InputFile file = (InputFile) value;
currentInputFiles.computeIfAbsent(file.getPartition(), k -> new ArrayList<>()).add(file.getFile());
} else if (value instanceof EndCheckpoint) {
EndCheckpoint endCheckpoint = (EndCheckpoint) value;
if (inputTaskTracker == null) {
inputTaskTracker = new TaskTracker(endCheckpoint.getNumberOfTasks());
}
// ensure all files are ready to be compacted.
boolean triggerCommit = inputTaskTracker.add(endCheckpoint.getCheckpointId(), endCheckpoint.getTaskId());
if (triggerCommit) {
commitUpToCheckpoint(endCheckpoint.getCheckpointId());
}
} else {
throw new UnsupportedOperationException("Unsupported input message: " + value);
}
}
use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.InputFile in project flink by apache.
the class CompactFileWriterTest method testEmitEndCheckpointAfterEndInput.
@Test
public void testEmitEndCheckpointAfterEndInput() throws Exception {
CompactFileWriter<RowData> compactFileWriter = new CompactFileWriter<>(1000, StreamingFileSink.forRowFormat(folder, new SimpleStringEncoder<>()));
try (OneInputStreamOperatorTestHarness<RowData, CoordinatorInput> harness = new OneInputStreamOperatorTestHarness<>(compactFileWriter)) {
harness.setup();
harness.open();
harness.processElement(row("test"), 0);
harness.snapshot(1, 1);
harness.notifyOfCompletedCheckpoint(1);
List<CoordinatorInput> coordinatorInputs = harness.extractOutputValues();
Assert.assertEquals(2, coordinatorInputs.size());
// assert emit InputFile
Assert.assertTrue(coordinatorInputs.get(0) instanceof InputFile);
// assert emit EndCheckpoint
Assert.assertEquals(1, ((EndCheckpoint) coordinatorInputs.get(1)).getCheckpointId());
harness.processElement(row("test1"), 0);
harness.processElement(row("test2"), 0);
harness.getOutput().clear();
// end input
harness.endInput();
coordinatorInputs = harness.extractOutputValues();
// assert emit EndCheckpoint with Long.MAX_VALUE lastly
EndCheckpoint endCheckpoint = (EndCheckpoint) coordinatorInputs.get(coordinatorInputs.size() - 1);
Assert.assertEquals(Long.MAX_VALUE, endCheckpoint.getCheckpointId());
}
}
use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.InputFile in project flink by apache.
the class CompactCoordinatorTest method testCoordinatorCrossCheckpoints.
@Test
public void testCoordinatorCrossCheckpoints() throws Exception {
AtomicReference<OperatorSubtaskState> state = new AtomicReference<>();
runCoordinator(harness -> {
harness.setup();
harness.open();
harness.processElement(new InputFile("p0", newFile("f0", 3)), 0);
harness.processElement(new InputFile("p0", newFile("f1", 2)), 0);
harness.processElement(new InputFile("p1", newFile("f2", 2)), 0);
harness.processElement(new InputFile("p0", newFile("f3", 5)), 0);
harness.processElement(new InputFile("p0", newFile("f4", 1)), 0);
harness.processElement(new InputFile("p1", newFile("f5", 5)), 0);
harness.processElement(new InputFile("p1", newFile("f6", 4)), 0);
state.set(harness.snapshot(1, 0));
});
runCoordinator(harness -> {
harness.setup();
harness.initializeState(state.get());
harness.open();
harness.processElement(new InputFile("p0", newFile("f7", 3)), 0);
harness.processElement(new InputFile("p0", newFile("f8", 2)), 0);
state.set(harness.snapshot(2, 0));
});
runCoordinator(harness -> {
harness.setup();
harness.initializeState(state.get());
harness.open();
harness.processElement(new EndCheckpoint(2, 0, 1), 0);
List<CoordinatorOutput> outputs = harness.extractOutputValues();
Assert.assertEquals(7, outputs.size());
List<CompactionUnit> cp1Units = new ArrayList<>();
for (int i = 0; i < 4; i++) {
CoordinatorOutput output = outputs.get(i);
Assert.assertTrue(output instanceof CompactionUnit);
cp1Units.add((CompactionUnit) output);
}
cp1Units.sort(Comparator.comparing(CompactionUnit::getPartition).thenComparingInt(CompactionUnit::getUnitId));
assertUnit(cp1Units.get(0), 0, "p0", Arrays.asList("f0", "f1", "f4"));
assertUnit(cp1Units.get(1), 1, "p0", Collections.singletonList("f3"));
assertUnit(cp1Units.get(2), 2, "p1", Arrays.asList("f2", "f5"));
assertUnit(cp1Units.get(3), 3, "p1", Collections.singletonList("f6"));
assertEndCompaction(outputs.get(4), 1);
assertUnit(outputs.get(5), 0, "p0", Arrays.asList("f7", "f8"));
assertEndCompaction(outputs.get(6), 2);
});
}
Aggregations