Examples with StreamReadMonitoringFunction org.apache.hudi.source.StreamReadMonitoringFunction used on opensource projects

Example 1 with StreamReadMonitoringFunction

use of org.apache.hudi.source.StreamReadMonitoringFunction in project hudi by apache.

the class HoodieTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
    return new DataStreamScanProvider() {

        @Override
        public boolean isBounded() {
            return !conf.getBoolean(FlinkOptions.READ_AS_STREAMING);
        }

        @Override
        public DataStream<RowData> produceDataStream(StreamExecutionEnvironment execEnv) {
            @SuppressWarnings("unchecked") TypeInformation<RowData> typeInfo = (TypeInformation<RowData>) TypeInfoDataTypeConverter.fromDataTypeToTypeInfo(getProducedDataType());
            if (conf.getBoolean(FlinkOptions.READ_AS_STREAMING)) {
                StreamReadMonitoringFunction monitoringFunction = new StreamReadMonitoringFunction(conf, FilePathUtils.toFlinkPath(path), maxCompactionMemoryInBytes, getRequiredPartitionPaths());
                InputFormat<RowData, ?> inputFormat = getInputFormat(true);
                OneInputStreamOperatorFactory<MergeOnReadInputSplit, RowData> factory = StreamReadOperator.factory((MergeOnReadInputFormat) inputFormat);
                SingleOutputStreamOperator<RowData> source = execEnv.addSource(monitoringFunction, getSourceOperatorName("split_monitor")).setParallelism(1).transform("split_reader", typeInfo, factory).setParallelism(conf.getInteger(FlinkOptions.READ_TASKS));
                return new DataStreamSource<>(source);
            } else {
                InputFormatSourceFunction<RowData> func = new InputFormatSourceFunction<>(getInputFormat(), typeInfo);
                DataStreamSource<RowData> source = execEnv.addSource(func, asSummaryString(), typeInfo);
                return source.name(getSourceOperatorName("bounded_source")).setParallelism(conf.getInteger(FlinkOptions.READ_TASKS));
            }
        }
    };
}