Examples with FileSplitAssigner org.apache.flink.connector.file.src.assigners.FileSplitAssigner used on opensource projects

Example 1 with FileSplitAssigner

use of org.apache.flink.connector.file.src.assigners.FileSplitAssigner in project flink by apache.

the class AbstractFileSource method createSplitEnumerator.

// ------------------------------------------------------------------------
// helpers
// ------------------------------------------------------------------------
private SplitEnumerator<SplitT, PendingSplitsCheckpoint<SplitT>> createSplitEnumerator(SplitEnumeratorContext<SplitT> context, FileEnumerator enumerator, Collection<FileSourceSplit> splits, @Nullable Collection<Path> alreadyProcessedPaths) {
    // cast this to a collection of FileSourceSplit because the enumerator code work
    // non-generically just on that base split type
    @SuppressWarnings("unchecked") final SplitEnumeratorContext<FileSourceSplit> fileSplitContext = (SplitEnumeratorContext<FileSourceSplit>) context;
    final FileSplitAssigner splitAssigner = assignerFactory.create(splits);
    if (continuousEnumerationSettings == null) {
        // bounded case
        return castGeneric(new StaticFileSplitEnumerator(fileSplitContext, splitAssigner));
    } else {
        // unbounded case
        if (alreadyProcessedPaths == null) {
            alreadyProcessedPaths = splitsToPaths(splits);
        }
        return castGeneric(new ContinuousFileSplitEnumerator(fileSplitContext, enumerator, splitAssigner, inputPaths, alreadyProcessedPaths, continuousEnumerationSettings.getDiscoveryInterval().toMillis()));
    }
}