Examples with ResultPartitionDeploymentDescriptor org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor used on opensource projects

Example 6 with ResultPartitionDeploymentDescriptor

use of org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor in project flink by apache.

the class ExecutionVertex method createDeploymentDescriptor.

/**
	 * Creates a task deployment descriptor to deploy a subtask to the given target slot.
	 *
	 * TODO: This should actually be in the EXECUTION
	 */
TaskDeploymentDescriptor createDeploymentDescriptor(ExecutionAttemptID executionId, SimpleSlot targetSlot, TaskStateHandles taskStateHandles, int attemptNumber) throws ExecutionGraphException {
    // Produced intermediate results
    List<ResultPartitionDeploymentDescriptor> producedPartitions = new ArrayList<>(resultPartitions.size());
    // Consumed intermediate results
    List<InputGateDeploymentDescriptor> consumedPartitions = new ArrayList<>(inputEdges.length);
    boolean lazyScheduling = getExecutionGraph().getScheduleMode().allowLazyDeployment();
    for (IntermediateResultPartition partition : resultPartitions.values()) {
        List<List<ExecutionEdge>> consumers = partition.getConsumers();
        if (consumers.isEmpty()) {
            //TODO this case only exists for test, currently there has to be exactly one consumer in real jobs!
            producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, ExecutionConfig.UPPER_BOUND_MAX_PARALLELISM, lazyScheduling));
        } else {
            Preconditions.checkState(1 == consumers.size(), "Only one consumer supported in the current implementation! Found: " + consumers.size());
            List<ExecutionEdge> consumer = consumers.get(0);
            ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex();
            int maxParallelism = vertex.getMaxParallelism();
            producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling));
        }
    }
    for (ExecutionEdge[] edges : inputEdges) {
        InputChannelDeploymentDescriptor[] partitions = InputChannelDeploymentDescriptor.fromEdges(edges, targetSlot, lazyScheduling);
        // If the produced partition has multiple consumers registered, we
        // need to request the one matching our sub task index.
        // TODO Refactor after removing the consumers from the intermediate result partitions
        int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size();
        int queueToRequest = subTaskIndex % numConsumerEdges;
        IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult();
        final IntermediateDataSetID resultId = consumedIntermediateResult.getId();
        final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
        consumedPartitions.add(new InputGateDeploymentDescriptor(resultId, partitionType, queueToRequest, partitions));
    }
    SerializedValue<JobInformation> serializedJobInformation = getExecutionGraph().getSerializedJobInformation();
    SerializedValue<TaskInformation> serializedJobVertexInformation = null;
    try {
        serializedJobVertexInformation = jobVertex.getSerializedTaskInformation();
    } catch (IOException e) {
        throw new ExecutionGraphException("Could not create a serialized JobVertexInformation for " + jobVertex.getJobVertexId(), e);
    }
    return new TaskDeploymentDescriptor(serializedJobInformation, serializedJobVertexInformation, executionId, targetSlot.getAllocatedSlot().getSlotAllocationId(), subTaskIndex, attemptNumber, targetSlot.getRoot().getSlotNumber(), taskStateHandles, producedPartitions, consumedPartitions);
}

Example 7 with ResultPartitionDeploymentDescriptor

use of org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor in project flink by apache.

the class TaskManagerTest method testRunJobWithForwardChannel.

@Test
public void testRunJobWithForwardChannel() {
    new JavaTestKit(system) {

        {
            ActorGateway jobManager = null;
            ActorGateway taskManager = null;
            final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), leaderSessionID);
            try {
                final JobID jid = new JobID();
                JobVertexID vid1 = new JobVertexID();
                JobVertexID vid2 = new JobVertexID();
                final ExecutionAttemptID eid1 = new ExecutionAttemptID();
                final ExecutionAttemptID eid2 = new ExecutionAttemptID();
                ActorRef jm = system.actorOf(Props.create(new SimpleLookupJobManagerCreator(leaderSessionID)));
                jobManager = new AkkaActorGateway(jm, leaderSessionID);
                taskManager = TestingUtils.createTaskManager(system, jobManager, new Configuration(), true, true);
                final ActorGateway tm = taskManager;
                IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID();
                List<ResultPartitionDeploymentDescriptor> irpdd = new ArrayList<ResultPartitionDeploymentDescriptor>();
                irpdd.add(new ResultPartitionDeploymentDescriptor(new IntermediateDataSetID(), partitionId, ResultPartitionType.PIPELINED, 1, 1, true));
                InputGateDeploymentDescriptor ircdd = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, new InputChannelDeploymentDescriptor[] { new InputChannelDeploymentDescriptor(new ResultPartitionID(partitionId, eid1), ResultPartitionLocation.createLocal()) });
                final TaskDeploymentDescriptor tdd1 = createTaskDeploymentDescriptor(jid, "TestJob", vid1, eid1, new SerializedValue<>(new ExecutionConfig()), "Sender", 1, 0, 1, 0, new Configuration(), new Configuration(), Tasks.Sender.class.getName(), irpdd, Collections.<InputGateDeploymentDescriptor>emptyList(), new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
                final TaskDeploymentDescriptor tdd2 = createTaskDeploymentDescriptor(jid, "TestJob", vid2, eid2, new SerializedValue<>(new ExecutionConfig()), "Receiver", 7, 2, 7, 0, new Configuration(), new Configuration(), Tasks.Receiver.class.getName(), Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.singletonList(ircdd), new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
                new Within(d) {

                    @Override
                    protected void run() {
                        try {
                            Future<Object> t1Running = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(eid1), timeout);
                            Future<Object> t2Running = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(eid2), timeout);
                            // submit the sender task
                            tm.tell(new SubmitTask(tdd1), testActorGateway);
                            expectMsgEquals(Acknowledge.get());
                            // wait until the sender task is running
                            Await.ready(t1Running, d);
                            // only now (after the sender is running), submit the receiver task
                            tm.tell(new SubmitTask(tdd2), testActorGateway);
                            expectMsgEquals(Acknowledge.get());
                            // wait until the receiver task is running
                            Await.ready(t2Running, d);
                            tm.tell(TestingTaskManagerMessages.getRequestRunningTasksMessage(), testActorGateway);
                            Map<ExecutionAttemptID, Task> tasks = expectMsgClass(TestingTaskManagerMessages.ResponseRunningTasks.class).asJava();
                            Task t1 = tasks.get(eid1);
                            Task t2 = tasks.get(eid2);
                            // we get to the check, so we need to guard the check
                            if (t1 != null) {
                                Future<Object> response = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskRemoved(eid1), timeout);
                                Await.ready(response, d);
                            }
                            if (t2 != null) {
                                Future<Object> response = tm.ask(new TestingTaskManagerMessages.NotifyWhenTaskRemoved(eid2), timeout);
                                Await.ready(response, d);
                                assertEquals(ExecutionState.FINISHED, t2.getExecutionState());
                            }
                            tm.tell(TestingTaskManagerMessages.getRequestRunningTasksMessage(), testActorGateway);
                            tasks = expectMsgClass(TestingTaskManagerMessages.ResponseRunningTasks.class).asJava();
                            assertEquals(0, tasks.size());
                        } catch (Exception e) {
                            e.printStackTrace();
                            fail(e.getMessage());
                        }
                    }
                };
            } catch (Exception e) {
                e.printStackTrace();
                fail(e.getMessage());
            } finally {
                // shut down the actors
                TestingUtils.stopActor(taskManager);
                TestingUtils.stopActor(jobManager);
            }
        }
    };
}