Examples with TaskTracker org.apache.flink.connector.file.table.stream.TaskTracker used on opensource projects

Example 1 with TaskTracker

use of org.apache.flink.connector.file.table.stream.TaskTracker in project flink by apache.

the class CompactCoordinator method processElement.

@Override
public void processElement(StreamRecord<CoordinatorInput> element) throws Exception {
    CoordinatorInput value = element.getValue();
    if (value instanceof InputFile) {
        InputFile file = (InputFile) value;
        currentInputFiles.computeIfAbsent(file.getPartition(), k -> new ArrayList<>()).add(file.getFile());
    } else if (value instanceof EndCheckpoint) {
        EndCheckpoint endCheckpoint = (EndCheckpoint) value;
        if (inputTaskTracker == null) {
            inputTaskTracker = new TaskTracker(endCheckpoint.getNumberOfTasks());
        }
        // ensure all files are ready to be compacted.
        boolean triggerCommit = inputTaskTracker.add(endCheckpoint.getCheckpointId(), endCheckpoint.getTaskId());
        if (triggerCommit) {
            commitUpToCheckpoint(endCheckpoint.getCheckpointId());
        }
    } else {
        throw new UnsupportedOperationException("Unsupported input message: " + value);
    }
}