Examples with CoordinatorInput org.apache.flink.connector.file.table.stream.compact.CompactMessages.CoordinatorInput used on opensource projects

Example 1 with CoordinatorInput

use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.CoordinatorInput in project flink by apache.

the class CompactCoordinator method processElement.

@Override
public void processElement(StreamRecord<CoordinatorInput> element) throws Exception {
    CoordinatorInput value = element.getValue();
    if (value instanceof InputFile) {
        InputFile file = (InputFile) value;
        currentInputFiles.computeIfAbsent(file.getPartition(), k -> new ArrayList<>()).add(file.getFile());
    } else if (value instanceof EndCheckpoint) {
        EndCheckpoint endCheckpoint = (EndCheckpoint) value;
        if (inputTaskTracker == null) {
            inputTaskTracker = new TaskTracker(endCheckpoint.getNumberOfTasks());
        }
        // ensure all files are ready to be compacted.
        boolean triggerCommit = inputTaskTracker.add(endCheckpoint.getCheckpointId(), endCheckpoint.getTaskId());
        if (triggerCommit) {
            commitUpToCheckpoint(endCheckpoint.getCheckpointId());
        }
    } else {
        throw new UnsupportedOperationException("Unsupported input message: " + value);
    }
}

Example 2 with CoordinatorInput

use of org.apache.flink.connector.file.table.stream.compact.CompactMessages.CoordinatorInput in project flink by apache.

the class CompactFileWriterTest method testEmitEndCheckpointAfterEndInput.

@Test
public void testEmitEndCheckpointAfterEndInput() throws Exception {
    CompactFileWriter<RowData> compactFileWriter = new CompactFileWriter<>(1000, StreamingFileSink.forRowFormat(folder, new SimpleStringEncoder<>()));
    try (OneInputStreamOperatorTestHarness<RowData, CoordinatorInput> harness = new OneInputStreamOperatorTestHarness<>(compactFileWriter)) {
        harness.setup();
        harness.open();
        harness.processElement(row("test"), 0);
        harness.snapshot(1, 1);
        harness.notifyOfCompletedCheckpoint(1);
        List<CoordinatorInput> coordinatorInputs = harness.extractOutputValues();
        Assert.assertEquals(2, coordinatorInputs.size());
        // assert emit InputFile
        Assert.assertTrue(coordinatorInputs.get(0) instanceof InputFile);
        // assert emit EndCheckpoint
        Assert.assertEquals(1, ((EndCheckpoint) coordinatorInputs.get(1)).getCheckpointId());
        harness.processElement(row("test1"), 0);
        harness.processElement(row("test2"), 0);
        harness.getOutput().clear();
        // end input
        harness.endInput();
        coordinatorInputs = harness.extractOutputValues();
        // assert emit EndCheckpoint with Long.MAX_VALUE lastly
        EndCheckpoint endCheckpoint = (EndCheckpoint) coordinatorInputs.get(coordinatorInputs.size() - 1);
        Assert.assertEquals(Long.MAX_VALUE, endCheckpoint.getCheckpointId());
    }
}