Examples with InputOutputFormatVertex org.apache.flink.runtime.jobgraph.InputOutputFormatVertex used on opensource projects

Example 1 with InputOutputFormatVertex

use of org.apache.flink.runtime.jobgraph.InputOutputFormatVertex in project flink by apache.

the class StreamingJobGraphGeneratorTest method testInputOutputFormat.

@Test
public void testInputOutputFormat() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> source = env.addSource(new InputFormatSourceFunction<>(new TypeSerializerInputFormat<>(TypeInformation.of(Long.class)), TypeInformation.of(Long.class)), TypeInformation.of(Long.class)).name("source");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink1");
    source.writeUsingOutputFormat(new DiscardingOutputFormat<>()).name("sink2");
    StreamGraph streamGraph = env.getStreamGraph();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    assertEquals(1, jobGraph.getNumberOfVertices());
    JobVertex jobVertex = jobGraph.getVertices().iterator().next();
    assertTrue(jobVertex instanceof InputOutputFormatVertex);
    InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(new TaskConfig(jobVertex.getConfiguration()), Thread.currentThread().getContextClassLoader());
    Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formatContainer.getInputFormats();
    Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
    assertEquals(1, inputFormats.size());
    assertEquals(2, outputFormats.size());
    Map<String, OperatorID> nameToOperatorIds = new HashMap<>();
    StreamConfig headConfig = new StreamConfig(jobVertex.getConfiguration());
    nameToOperatorIds.put(headConfig.getOperatorName(), headConfig.getOperatorID());
    Map<Integer, StreamConfig> chainedConfigs = headConfig.getTransitiveChainedTaskConfigs(Thread.currentThread().getContextClassLoader());
    for (StreamConfig config : chainedConfigs.values()) {
        nameToOperatorIds.put(config.getOperatorName(), config.getOperatorID());
    }
    InputFormat<?, ?> sourceFormat = inputFormats.get(nameToOperatorIds.get("Source: source")).getUserCodeObject();
    assertTrue(sourceFormat instanceof TypeSerializerInputFormat);
    OutputFormat<?> sinkFormat1 = outputFormats.get(nameToOperatorIds.get("Sink: sink1")).getUserCodeObject();
    assertTrue(sinkFormat1 instanceof DiscardingOutputFormat);
    OutputFormat<?> sinkFormat2 = outputFormats.get(nameToOperatorIds.get("Sink: sink2")).getUserCodeObject();
    assertTrue(sinkFormat2 instanceof DiscardingOutputFormat);
}

Example 2 with InputOutputFormatVertex

use of org.apache.flink.runtime.jobgraph.InputOutputFormatVertex in project flink by apache.

the class JobGraphGenerator method createDataSinkVertex.

private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
    final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
    final TaskConfig config = new TaskConfig(vertex.getConfiguration());
    final OperatorID operatorID = new OperatorID();
    vertex.setResources(node.getMinResources(), node.getPreferredResources());
    vertex.setInvokableClass(DataSinkTask.class);
    vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
    // set user code
    new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader()).addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper()).addParameters(operatorID, node.getProgramOperator().getParameters()).write(config);
    return vertex;
}

Example 3 with InputOutputFormatVertex

use of org.apache.flink.runtime.jobgraph.InputOutputFormatVertex in project flink by apache.

the class JobGraphGenerator method createDataSourceVertex.

private JobVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
    final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
    final TaskConfig config = new TaskConfig(vertex.getConfiguration());
    final OperatorID operatorID = new OperatorID();
    vertex.setResources(node.getMinResources(), node.getPreferredResources());
    vertex.setInvokableClass(DataSourceTask.class);
    vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
    // set user code
    new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader()).addInputFormat(operatorID, (UserCodeWrapper<? extends InputFormat<?, ?>>) node.getProgramOperator().getUserCodeWrapper()).addParameters(operatorID, node.getProgramOperator().getParameters()).write(config);
    config.setOutputSerializer(node.getSerializer());
    return vertex;
}

Example 4 with InputOutputFormatVertex

use of org.apache.flink.runtime.jobgraph.InputOutputFormatVertex in project flink by apache.

the class StreamingJobGraphGenerator method createJobVertex.

private StreamConfig createJobVertex(Integer streamNodeId, OperatorChainInfo chainInfo) {
    JobVertex jobVertex;
    StreamNode streamNode = streamGraph.getStreamNode(streamNodeId);
    byte[] hash = chainInfo.getHash(streamNodeId);
    if (hash == null) {
        throw new IllegalStateException("Cannot find node hash. " + "Did you generate them before calling this method?");
    }
    JobVertexID jobVertexId = new JobVertexID(hash);
    List<Tuple2<byte[], byte[]>> chainedOperators = chainInfo.getChainedOperatorHashes(streamNodeId);
    List<OperatorIDPair> operatorIDPairs = new ArrayList<>();
    if (chainedOperators != null) {
        for (Tuple2<byte[], byte[]> chainedOperator : chainedOperators) {
            OperatorID userDefinedOperatorID = chainedOperator.f1 == null ? null : new OperatorID(chainedOperator.f1);
            operatorIDPairs.add(OperatorIDPair.of(new OperatorID(chainedOperator.f0), userDefinedOperatorID));
        }
    }
    if (chainedInputOutputFormats.containsKey(streamNodeId)) {
        jobVertex = new InputOutputFormatVertex(chainedNames.get(streamNodeId), jobVertexId, operatorIDPairs);
        chainedInputOutputFormats.get(streamNodeId).write(new TaskConfig(jobVertex.getConfiguration()));
    } else {
        jobVertex = new JobVertex(chainedNames.get(streamNodeId), jobVertexId, operatorIDPairs);
    }
    for (OperatorCoordinator.Provider coordinatorProvider : chainInfo.getCoordinatorProviders()) {
        try {
            jobVertex.addOperatorCoordinator(new SerializedValue<>(coordinatorProvider));
        } catch (IOException e) {
            throw new FlinkRuntimeException(String.format("Coordinator Provider for node %s is not serializable.", chainedNames.get(streamNodeId)), e);
        }
    }
    jobVertex.setResources(chainedMinResources.get(streamNodeId), chainedPreferredResources.get(streamNodeId));
    jobVertex.setInvokableClass(streamNode.getJobVertexClass());
    int parallelism = streamNode.getParallelism();
    if (parallelism > 0) {
        jobVertex.setParallelism(parallelism);
    } else {
        parallelism = jobVertex.getParallelism();
    }
    jobVertex.setMaxParallelism(streamNode.getMaxParallelism());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Parallelism set: {} for {}", parallelism, streamNodeId);
    }
    jobVertices.put(streamNodeId, jobVertex);
    builtVertices.add(streamNodeId);
    jobGraph.addVertex(jobVertex);
    return new StreamConfig(jobVertex.getConfiguration());
}