Examples with ExecutionJobVertex org.apache.flink.runtime.executiongraph.ExecutionJobVertex used on opensource projects

Example 36 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class JobMaster method requestNextInputSplit.

@RpcMethod
public SerializedInputSplit requestNextInputSplit(final UUID leaderSessionID, final JobVertexID vertexID, final ExecutionAttemptID executionAttempt) throws Exception {
    validateLeaderSessionId(leaderSessionID);
    final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt);
    if (execution == null) {
        // but TaskManager get some delay to aware of that situation
        if (log.isDebugEnabled()) {
            log.debug("Can not find Execution for attempt {}.", executionAttempt);
        }
        // but we should TaskManager be aware of this
        throw new Exception("Can not find Execution for attempt " + executionAttempt);
    }
    final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID);
    if (vertex == null) {
        log.error("Cannot find execution vertex for vertex ID {}.", vertexID);
        throw new Exception("Cannot find execution vertex for vertex ID " + vertexID);
    }
    final InputSplitAssigner splitAssigner = vertex.getSplitAssigner();
    if (splitAssigner == null) {
        log.error("No InputSplitAssigner for vertex ID {}.", vertexID);
        throw new Exception("No InputSplitAssigner for vertex ID " + vertexID);
    }
    final Slot slot = execution.getAssignedResource();
    final int taskId = execution.getVertex().getParallelSubtaskIndex();
    final String host = slot != null ? slot.getTaskManagerLocation().getHostname() : null;
    final InputSplit nextInputSplit = splitAssigner.getNextInputSplit(host, taskId);
    if (log.isDebugEnabled()) {
        log.debug("Send next input split {}.", nextInputSplit);
    }
    try {
        final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit);
        return new SerializedInputSplit(serializedInputSplit);
    } catch (Exception ex) {
        log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex);
        IOException reason = new IOException("Could not serialize the next input split of class " + nextInputSplit.getClass() + ".", ex);
        vertex.fail(reason);
        throw reason;
    }
}

Example 37 with ExecutionJobVertex

use of org.apache.flink.runtime.executiongraph.ExecutionJobVertex in project flink by apache.

the class RescalePartitionerTest method testExecutionGraphGeneration.

@Test
public void testExecutionGraphGeneration() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    // get input data
    DataStream<String> text = env.addSource(new ParallelSourceFunction<String>() {

        private static final long serialVersionUID = 7772338606389180774L;

        @Override
        public void run(SourceContext<String> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    }).setParallelism(2);
    DataStream<Tuple2<String, Integer>> counts = text.rescale().flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {

        private static final long serialVersionUID = -5255930322161596829L;

        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
        }
    });
    counts.rescale().print().setParallelism(2);
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    final JobID jobId = new JobID();
    final String jobName = "Semi-Rebalance Test Job";
    final Configuration cfg = new Configuration();
    List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    JobVertex sourceVertex = jobVertices.get(0);
    JobVertex mapVertex = jobVertices.get(1);
    JobVertex sinkVertex = jobVertices.get(2);
    assertEquals(2, sourceVertex.getParallelism());
    assertEquals(4, mapVertex.getParallelism());
    assertEquals(2, sinkVertex.getParallelism());
    ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new ArrayList<BlobKey>(), new ArrayList<URL>(), new Scheduler(TestingUtils.defaultExecutionContext()), ExecutionGraph.class.getClassLoader(), new UnregisteredMetricsGroup());
    try {
        eg.attachJobGraph(jobVertices);
    } catch (JobException e) {
        e.printStackTrace();
        fail("Building ExecutionGraph failed: " + e.getMessage());
    }
    ExecutionJobVertex execSourceVertex = eg.getJobVertex(sourceVertex.getID());
    ExecutionJobVertex execMapVertex = eg.getJobVertex(mapVertex.getID());
    ExecutionJobVertex execSinkVertex = eg.getJobVertex(sinkVertex.getID());
    assertEquals(0, execSourceVertex.getInputs().size());
    assertEquals(1, execMapVertex.getInputs().size());
    assertEquals(4, execMapVertex.getParallelism());
    ExecutionVertex[] mapTaskVertices = execMapVertex.getTaskVertices();
    // verify that we have each parallel input partition exactly twice, i.e. that one source
    // sends to two unique mappers
    Map<Integer, Integer> mapInputPartitionCounts = new HashMap<>();
    for (ExecutionVertex mapTaskVertex : mapTaskVertices) {
        assertEquals(1, mapTaskVertex.getNumberOfInputs());
        assertEquals(1, mapTaskVertex.getInputEdges(0).length);
        ExecutionEdge inputEdge = mapTaskVertex.getInputEdges(0)[0];
        assertEquals(sourceVertex.getID(), inputEdge.getSource().getProducer().getJobvertexId());
        int inputPartition = inputEdge.getSource().getPartitionNumber();
        if (!mapInputPartitionCounts.containsKey(inputPartition)) {
            mapInputPartitionCounts.put(inputPartition, 1);
        } else {
            mapInputPartitionCounts.put(inputPartition, mapInputPartitionCounts.get(inputPartition) + 1);
        }
    }
    assertEquals(2, mapInputPartitionCounts.size());
    for (int count : mapInputPartitionCounts.values()) {
        assertEquals(2, count);
    }
    assertEquals(1, execSinkVertex.getInputs().size());
    assertEquals(2, execSinkVertex.getParallelism());
    ExecutionVertex[] sinkTaskVertices = execSinkVertex.getTaskVertices();
    // verify each sink instance has two inputs from the map and that each map subpartition
    // only occurs in one unique input edge
    Set<Integer> mapSubpartitions = new HashSet<>();
    for (ExecutionVertex sinkTaskVertex : sinkTaskVertices) {
        assertEquals(1, sinkTaskVertex.getNumberOfInputs());
        assertEquals(2, sinkTaskVertex.getInputEdges(0).length);
        ExecutionEdge inputEdge1 = sinkTaskVertex.getInputEdges(0)[0];
        ExecutionEdge inputEdge2 = sinkTaskVertex.getInputEdges(0)[1];
        assertEquals(mapVertex.getID(), inputEdge1.getSource().getProducer().getJobvertexId());
        assertEquals(mapVertex.getID(), inputEdge2.getSource().getProducer().getJobvertexId());
        int inputPartition1 = inputEdge1.getSource().getPartitionNumber();
        assertFalse(mapSubpartitions.contains(inputPartition1));
        mapSubpartitions.add(inputPartition1);
        int inputPartition2 = inputEdge2.getSource().getPartitionNumber();
        assertFalse(mapSubpartitions.contains(inputPartition2));
        mapSubpartitions.add(inputPartition2);
    }
    assertEquals(4, mapSubpartitions.size());
}