Search in sources :

Example 1 with TransformationSinkProvider

use of org.apache.flink.table.planner.connectors.TransformationSinkProvider in project flink by apache.

the class CommonExecSink method applySinkProvider.

private Transformation<?> applySinkProvider(Transformation<RowData> inputTransform, StreamExecutionEnvironment env, SinkRuntimeProvider runtimeProvider, int rowtimeFieldIndex, int sinkParallelism, ReadableConfig config) {
    TransformationMetadata sinkMeta = createTransformationMeta(SINK_TRANSFORMATION, config);
    if (runtimeProvider instanceof DataStreamSinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final DataStreamSinkProvider provider = (DataStreamSinkProvider) runtimeProvider;
        return provider.consumeDataStream(createProviderContext(), dataStream).getTransformation();
    } else if (runtimeProvider instanceof TransformationSinkProvider) {
        final TransformationSinkProvider provider = (TransformationSinkProvider) runtimeProvider;
        return provider.createTransformation(new TransformationSinkProvider.Context() {

            @Override
            public Transformation<RowData> getInputTransformation() {
                return inputTransform;
            }

            @Override
            public int getRowtimeIndex() {
                return rowtimeFieldIndex;
            }

            @Override
            public Optional<String> generateUid(String name) {
                return createProviderContext().generateUid(name);
            }
        });
    } else if (runtimeProvider instanceof SinkFunctionProvider) {
        final SinkFunction<RowData> sinkFunction = ((SinkFunctionProvider) runtimeProvider).createSinkFunction();
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof OutputFormatProvider) {
        OutputFormat<RowData> outputFormat = ((OutputFormatProvider) runtimeProvider).createOutputFormat();
        final SinkFunction<RowData> sinkFunction = new OutputFormatSinkFunction<>(outputFormat);
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof SinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSinkV1(dataStream, ((SinkProvider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else if (runtimeProvider instanceof SinkV2Provider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSink(dataStream, ((SinkV2Provider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else {
        throw new TableException("Unsupported sink runtime provider.");
    }
}
Also used : ExecNodeContext(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext) ProviderContext(org.apache.flink.table.connector.ProviderContext) SinkRuntimeProviderContext(org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext) TransformationMetadata(org.apache.flink.table.planner.plan.nodes.exec.utils.TransformationMetadata) PartitionTransformation(org.apache.flink.streaming.api.transformations.PartitionTransformation) LegacySinkTransformation(org.apache.flink.streaming.api.transformations.LegacySinkTransformation) OneInputTransformation(org.apache.flink.streaming.api.transformations.OneInputTransformation) Transformation(org.apache.flink.api.dag.Transformation) TableException(org.apache.flink.table.api.TableException) DataStream(org.apache.flink.streaming.api.datastream.DataStream) DataStreamSinkProvider(org.apache.flink.table.connector.sink.DataStreamSinkProvider) OutputFormat(org.apache.flink.api.common.io.OutputFormat) DataStreamSinkProvider(org.apache.flink.table.connector.sink.DataStreamSinkProvider) TransformationSinkProvider(org.apache.flink.table.planner.connectors.TransformationSinkProvider) SinkProvider(org.apache.flink.table.connector.sink.SinkProvider) SinkFunctionProvider(org.apache.flink.table.connector.sink.SinkFunctionProvider) RowData(org.apache.flink.table.data.RowData) TransformationSinkProvider(org.apache.flink.table.planner.connectors.TransformationSinkProvider) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) OutputFormatSinkFunction(org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction) SinkV2Provider(org.apache.flink.table.connector.sink.SinkV2Provider) OutputFormatProvider(org.apache.flink.table.connector.sink.OutputFormatProvider)

Aggregations

OutputFormat (org.apache.flink.api.common.io.OutputFormat)1 Transformation (org.apache.flink.api.dag.Transformation)1 DataStream (org.apache.flink.streaming.api.datastream.DataStream)1 OutputFormatSinkFunction (org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction)1 SinkFunction (org.apache.flink.streaming.api.functions.sink.SinkFunction)1 LegacySinkTransformation (org.apache.flink.streaming.api.transformations.LegacySinkTransformation)1 OneInputTransformation (org.apache.flink.streaming.api.transformations.OneInputTransformation)1 PartitionTransformation (org.apache.flink.streaming.api.transformations.PartitionTransformation)1 TableException (org.apache.flink.table.api.TableException)1 ProviderContext (org.apache.flink.table.connector.ProviderContext)1 DataStreamSinkProvider (org.apache.flink.table.connector.sink.DataStreamSinkProvider)1 OutputFormatProvider (org.apache.flink.table.connector.sink.OutputFormatProvider)1 SinkFunctionProvider (org.apache.flink.table.connector.sink.SinkFunctionProvider)1 SinkProvider (org.apache.flink.table.connector.sink.SinkProvider)1 SinkV2Provider (org.apache.flink.table.connector.sink.SinkV2Provider)1 RowData (org.apache.flink.table.data.RowData)1 TransformationSinkProvider (org.apache.flink.table.planner.connectors.TransformationSinkProvider)1 ExecNodeContext (org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext)1 TransformationMetadata (org.apache.flink.table.planner.plan.nodes.exec.utils.TransformationMetadata)1 SinkRuntimeProviderContext (org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext)1