Examples with ContinuousFileReaderOperatorFactory org.apache.flink.streaming.api.functions.source.ContinuousFileReaderOperatorFactory used on opensource projects

Example 1 with ContinuousFileReaderOperatorFactory

use of org.apache.flink.streaming.api.functions.source.ContinuousFileReaderOperatorFactory in project flink by apache.

the class StreamExecutionEnvironment method createFileInput.

private <OUT> DataStreamSource<OUT> createFileInput(FileInputFormat<OUT> inputFormat, TypeInformation<OUT> typeInfo, String sourceName, FileProcessingMode monitoringMode, long interval) {
    Preconditions.checkNotNull(inputFormat, "Unspecified file input format.");
    Preconditions.checkNotNull(typeInfo, "Unspecified output type information.");
    Preconditions.checkNotNull(sourceName, "Unspecified name for the source.");
    Preconditions.checkNotNull(monitoringMode, "Unspecified monitoring mode.");
    Preconditions.checkArgument(monitoringMode.equals(FileProcessingMode.PROCESS_ONCE) || interval >= ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL, "The path monitoring interval cannot be less than " + ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL + " ms.");
    ContinuousFileMonitoringFunction<OUT> monitoringFunction = new ContinuousFileMonitoringFunction<>(inputFormat, monitoringMode, getParallelism(), interval);
    ContinuousFileReaderOperatorFactory<OUT, TimestampedFileInputSplit> factory = new ContinuousFileReaderOperatorFactory<>(inputFormat);
    final Boundedness boundedness = monitoringMode == FileProcessingMode.PROCESS_ONCE ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED;
    SingleOutputStreamOperator<OUT> source = addSource(monitoringFunction, sourceName, null, boundedness).transform("Split Reader: " + sourceName, typeInfo, factory);
    return new DataStreamSource<>(source);
}