Examples with Boundedness org.apache.flink.api.connector.source.Boundedness used on opensource projects

Example 1 with Boundedness

use of org.apache.flink.api.connector.source.Boundedness in project flink by apache.

the class CommonExecTableSourceScan method createSourceFunctionTransformation.

/**
 * Adopted from {@link StreamExecutionEnvironment#addSource(SourceFunction, String,
 * TypeInformation)} but with custom {@link Boundedness}.
 */
protected Transformation<RowData> createSourceFunctionTransformation(StreamExecutionEnvironment env, SourceFunction<RowData> function, boolean isBounded, String operatorName, TypeInformation<RowData> outputTypeInfo) {
    env.clean(function);
    final int parallelism;
    if (function instanceof ParallelSourceFunction) {
        parallelism = env.getParallelism();
    } else {
        parallelism = 1;
    }
    final Boundedness boundedness;
    if (isBounded) {
        boundedness = Boundedness.BOUNDED;
    } else {
        boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
    }
    final StreamSource<RowData, ?> sourceOperator = new StreamSource<>(function, !isBounded);
    return new LegacySourceTransformation<>(operatorName, sourceOperator, outputTypeInfo, parallelism, boundedness);
}

Example 2 with Boundedness

use of org.apache.flink.api.connector.source.Boundedness in project flink by apache.

the class StreamExecutionEnvironment method createFileInput.

private <OUT> DataStreamSource<OUT> createFileInput(FileInputFormat<OUT> inputFormat, TypeInformation<OUT> typeInfo, String sourceName, FileProcessingMode monitoringMode, long interval) {
    Preconditions.checkNotNull(inputFormat, "Unspecified file input format.");
    Preconditions.checkNotNull(typeInfo, "Unspecified output type information.");
    Preconditions.checkNotNull(sourceName, "Unspecified name for the source.");
    Preconditions.checkNotNull(monitoringMode, "Unspecified monitoring mode.");
    Preconditions.checkArgument(monitoringMode.equals(FileProcessingMode.PROCESS_ONCE) || interval >= ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL, "The path monitoring interval cannot be less than " + ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL + " ms.");
    ContinuousFileMonitoringFunction<OUT> monitoringFunction = new ContinuousFileMonitoringFunction<>(inputFormat, monitoringMode, getParallelism(), interval);
    ContinuousFileReaderOperatorFactory<OUT, TimestampedFileInputSplit> factory = new ContinuousFileReaderOperatorFactory<>(inputFormat);
    final Boundedness boundedness = monitoringMode == FileProcessingMode.PROCESS_ONCE ? Boundedness.BOUNDED : Boundedness.CONTINUOUS_UNBOUNDED;
    SingleOutputStreamOperator<OUT> source = addSource(monitoringFunction, sourceName, null, boundedness).transform("Split Reader: " + sourceName, typeInfo, factory);
    return new DataStreamSource<>(source);
}