Examples with OutputFormat org.apache.flink.api.common.io.OutputFormat used on opensource projects

Example 6 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class InputOutputFormatContainerTest method testOnlyOutputFormat.

@Test
public void testOnlyOutputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID = new OperatorID();
    formatContainer.addOutputFormat(operatorID, new DiscardingOutputFormat<>());
    Configuration parameters = new Configuration();
    parameters.setString("parameter1", "bcd234");
    formatContainer.addParameters(operatorID, parameters);
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = loadedFormatContainer.getOutputFormats();
    assertEquals(1, outputFormats.size());
    assertEquals(0, loadedFormatContainer.getInputFormats().size());
    assertTrue(outputFormats.get(operatorID).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration loadedParameters = loadedFormatContainer.getParameters(operatorID);
    assertEquals(1, loadedParameters.keySet().size());
    assertEquals("bcd234", loadedParameters.getString("parameter1", null));
}

Example 7 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class InputOutputFormatContainerTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID1 = new OperatorID();
    formatContainer.addInputFormat(operatorID1, new TestInputFormat("test input format"));
    formatContainer.addParameters(operatorID1, "parameter1", "abc123");
    OperatorID operatorID2 = new OperatorID();
    formatContainer.addOutputFormat(operatorID2, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID2, "parameter1", "bcd234");
    OperatorID operatorID3 = new OperatorID();
    formatContainer.addOutputFormat(operatorID3, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID3, "parameter1", "cde345");
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = loadedFormatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    // verify the input format
    TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID1).getUserCodeObject();
    assertEquals("test input format", inputFormat.getName());
    Configuration inputFormatParams = loadedFormatContainer.getParameters(operatorID1);
    assertEquals(1, inputFormatParams.keySet().size());
    assertEquals("abc123", inputFormatParams.getString("parameter1", null));
    // verify the output formats
    assertTrue(outputFormats.get(operatorID2).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams1 = loadedFormatContainer.getParameters(operatorID2);
    assertEquals(1, outputFormatParams1.keySet().size());
    assertEquals("bcd234", outputFormatParams1.getString("parameter1", null));
    assertTrue(outputFormats.get(operatorID3).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams2 = loadedFormatContainer.getParameters(operatorID3);
    assertEquals(1, outputFormatParams2.keySet().size());
    assertEquals("cde345", outputFormatParams2.getString("parameter1", null));
}

Example 8 with OutputFormat

use of org.apache.flink.api.common.io.OutputFormat in project flink by apache.

the class CommonExecSink method applySinkProvider.

private Transformation<?> applySinkProvider(Transformation<RowData> inputTransform, StreamExecutionEnvironment env, SinkRuntimeProvider runtimeProvider, int rowtimeFieldIndex, int sinkParallelism, ReadableConfig config) {
    TransformationMetadata sinkMeta = createTransformationMeta(SINK_TRANSFORMATION, config);
    if (runtimeProvider instanceof DataStreamSinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final DataStreamSinkProvider provider = (DataStreamSinkProvider) runtimeProvider;
        return provider.consumeDataStream(createProviderContext(), dataStream).getTransformation();
    } else if (runtimeProvider instanceof TransformationSinkProvider) {
        final TransformationSinkProvider provider = (TransformationSinkProvider) runtimeProvider;
        return provider.createTransformation(new TransformationSinkProvider.Context() {

            @Override
            public Transformation<RowData> getInputTransformation() {
                return inputTransform;
            }

            @Override
            public int getRowtimeIndex() {
                return rowtimeFieldIndex;
            }

            @Override
            public Optional<String> generateUid(String name) {
                return createProviderContext().generateUid(name);
            }
        });
    } else if (runtimeProvider instanceof SinkFunctionProvider) {
        final SinkFunction<RowData> sinkFunction = ((SinkFunctionProvider) runtimeProvider).createSinkFunction();
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof OutputFormatProvider) {
        OutputFormat<RowData> outputFormat = ((OutputFormatProvider) runtimeProvider).createOutputFormat();
        final SinkFunction<RowData> sinkFunction = new OutputFormatSinkFunction<>(outputFormat);
        return createSinkFunctionTransformation(sinkFunction, env, inputTransform, rowtimeFieldIndex, sinkMeta, sinkParallelism);
    } else if (runtimeProvider instanceof SinkProvider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSinkV1(dataStream, ((SinkProvider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else if (runtimeProvider instanceof SinkV2Provider) {
        Transformation<RowData> sinkTransformation = applyRowtimeTransformation(inputTransform, rowtimeFieldIndex, sinkParallelism, config);
        final DataStream<RowData> dataStream = new DataStream<>(env, sinkTransformation);
        final Transformation<?> transformation = DataStreamSink.forSink(dataStream, ((SinkV2Provider) runtimeProvider).createSink()).getTransformation();
        transformation.setParallelism(sinkParallelism);
        sinkMeta.fill(transformation);
        return transformation;
    } else {
        throw new TableException("Unsupported sink runtime provider.");
    }
}