Examples with UserCodeWrapper org.apache.flink.api.common.operators.util.UserCodeWrapper used on opensource projects

Example 1 with UserCodeWrapper

use of org.apache.flink.api.common.operators.util.UserCodeWrapper in project flink by apache.

the class StreamingJobGraphGeneratorTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> source = env.addSource(new InputFormatSourceFunction<>(new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)), TypeInformation.of(Long.class)), TypeInformation.of(Long.class)).name("source");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");
    StreamGraph streamGraph = env.getStreamGraph();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    assertEquals(1, jobGraph.getNumberOfVertices());
    JobVertex jobVertex = jobGraph.getVertices().iterator().next();
    assertTrue(jobVertex instanceof InputOutputFormatVertex);
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
    StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
    nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());
    Map<Integer, StreamConfig> chainedConfigs = headConfig.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
    for (StreamConfig config : chainedConfigs.values()) {
        nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
    }
    InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
    assertTrue(sourceFormat instanceof TypeSerializerInputFormat);
    OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
    assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);
    OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
    assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}

Example 2 with UserCodeWrapper

use of org.apache.flink.api.common.operators.util.UserCodeWrapper in project flink by apache.

the class InputOutputFormatContainerTest method testOnlyInputFormat.

@Test
public void testOnlyInputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID = new OperatorID();
    formatContainer.addInputFormat(operatorID, new TestInputFormat("test input format"));
    formatContainer.addParameters(operatorID, "parameter1", "abc123");
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(0, loadedFormatContainer.getOutputFormats().size());
    TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID).getUserCodeObject();
    assertEquals("test input format", inputFormat.getName());
    Configuration parameters = loadedFormatContainer.getParameters(operatorID);
    assertEquals(1, parameters.keySet().size());
    assertEquals("abc123", parameters.getString("parameter1", null));
}

Example 3 with UserCodeWrapper

use of org.apache.flink.api.common.operators.util.UserCodeWrapper in project flink by apache.

the class InputOutputFormatContainerTest method testOnlyOutputFormat.

@Test
public void testOnlyOutputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID = new OperatorID();
    formatContainer.addOutputFormat(operatorID, new DiscardingOutputFormat<>());
    Configuration parameters = new Configuration();
    parameters.setString("parameter1", "bcd234");
    formatContainer.addParameters(operatorID, parameters);
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = loadedFormatContainer.getOutputFormats();
    assertEquals(1, outputFormats.size());
    assertEquals(0, loadedFormatContainer.getInputFormats().size());
    assertTrue(outputFormats.get(operatorID).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration loadedParameters = loadedFormatContainer.getParameters(operatorID);
    assertEquals(1, loadedParameters.keySet().size());
    assertEquals("bcd234", loadedParameters.getString("parameter1", null));
}

Example 4 with UserCodeWrapper

use of org.apache.flink.api.common.operators.util.UserCodeWrapper in project flink by apache.

the class InputOutputFormatContainerTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());
    OperatorID operatorID1 = new OperatorID();
    formatContainer.addInputFormat(operatorID1, new TestInputFormat("test input format"));
    formatContainer.addParameters(operatorID1, "parameter1", "abc123");
    OperatorID operatorID2 = new OperatorID();
    formatContainer.addOutputFormat(operatorID2, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID2, "parameter1", "bcd234");
    OperatorID operatorID3 = new OperatorID();
    formatContainer.addOutputFormat(operatorID3, new DiscardingOutputFormat());
    formatContainer.addParameters(operatorID3, "parameter1", "cde345");
    TaskConfig taskConfig = new TaskConfig(new Configuration());
    formatContainer.write(taskConfig);
    InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = loadedFormatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    // verify the input format
    TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID1).getUserCodeObject();
    assertEquals("test input format", inputFormat.getName());
    Configuration inputFormatParams = loadedFormatContainer.getParameters(operatorID1);
    assertEquals(1, inputFormatParams.keySet().size());
    assertEquals("abc123", inputFormatParams.getString("parameter1", null));
    // verify the output formats
    assertTrue(outputFormats.get(operatorID2).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams1 = loadedFormatContainer.getParameters(operatorID2);
    assertEquals(1, outputFormatParams1.keySet().size());
    assertEquals("bcd234", outputFormatParams1.getString("parameter1", null));
    assertTrue(outputFormats.get(operatorID3).getUserCodeObject() instanceof DiscardingOutputFormat);
    Configuration outputFormatParams2 = loadedFormatContainer.getParameters(operatorID3);
    assertEquals(1, outputFormatParams2.keySet().size());
    assertEquals("cde345", outputFormatParams2.getString("parameter1", null));
}