Examples with OutputFormatSinkFunction org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction used on opensource projects

Example 1 with OutputFormatSinkFunction

use of org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction in project flink by apache.

the class FileSystemOutputFormatTest method createSink.

private OneInputStreamOperatorTestHarness<Row, Object> createSink(boolean override, boolean partition, boolean dynamicGrouped, LinkedHashMap<String, String> staticPartitions, AtomicReference<FileSystemOutputFormat<Row>> sinkRef) throws Exception {
    String[] columnNames = new String[] { "a", "b", "c" };
    String[] partitionColumns = partition ? new String[] { "c" } : new String[0];
    TableMetaStoreFactory msFactory = new FileSystemCommitterTest.TestMetaStoreFactory(new Path(outputFile.getPath()));
    FileSystemOutputFormat<Row> sink = new FileSystemOutputFormat.Builder<Row>().setMetaStoreFactory(msFactory).setTempPath(new Path(tmpFile.getPath())).setOverwrite(override).setPartitionColumns(partitionColumns).setPartitionComputer(new RowPartitionComputer("default", columnNames, partitionColumns)).setFormatFactory(TextOutputFormat::new).setDynamicGrouped(dynamicGrouped).setStaticPartitions(staticPartitions).build();
    sinkRef.set(sink);
    return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(new OutputFormatSinkFunction<>(sink)), // test parallelism
    3, 3, 0);
}

Example 2 with OutputFormatSinkFunction

use of org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction in project flink by apache.

the class CommonExecSink method applySinkProvider.

private Transformation<?> applySinkProvider(Transformation<RowData> inputTransform, StreamExecutionEnvironment env, SinkRuntimeProvider runtimeProvider, int rowtimeFieldIndex, int sinkParallelism, ReadableConfig config) {
    TransformationMetadata sinkMeta = createTransformationMeta(SINK_TRANSFORMATION, config);
    if (runtimeProvider instanceof DataStreamSinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final DataStreamSinkProvider provider = (DataStreamSinkProvider) runtimeProvider;
        return provider.consumeDataStream(createProviderContext(), dataStream).getTransformation();
    } else if (runtimeProvider instanceof TransformationSinkProvider) {
        final TransformationSinkProvider provider = (TransformationSinkProvider) runtimeProvider;
        return provider.createTransformation(new TransformationSinkProvider.Context() {

            @Override
            public Transformation<RowData> getInputTransformation() {
                return inputTransform;
            }

            @Override
            public int getRowtimeIndex() {
                return rowtimeFieldIndex;
            }

            @Override
            public Optional<String> generateUid(String name) {
                return createProviderContext().generateUid(name);
            }
        });
    } else if (runtimeProvider instanceof SinkFunctionProvider) {
        final SinkFunction<RowData> sinkFunction = ((SinkFunctionProvider) runtimeProvider).createSinkFunction();
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof OutputFormatProvider) {
        OutputFormat<RowData> outputFormat = ((OutputFormatProvider) runtimeProvider).createOutputFormat();
        final SinkFunction<RowData> sinkFunction = new OutputFormatSinkFunction<>(outputFormat);
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof SinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSinkV1(dataStream, ((SinkProvider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else if (runtimeProvider instanceof SinkV2Provider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSink(dataStream, ((SinkV2Provider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else {
        throw new TableException("Unsupported sink runtime provider.");
    }
}