Examples with IntermediateDataSetID org.apache.flink.runtime.jobgraph.IntermediateDataSetID used on opensource projects

Example 16 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class JobManagerTest method testRequestPartitionState.

/**
	 * Tests responses to partition state requests.
	 */
@Test
public void testRequestPartitionState() throws Exception {
    new JavaTestKit(system) {

        {
            new Within(duration("15 seconds")) {

                @Override
                protected void run() {
                    // Setup
                    TestingCluster cluster = null;
                    try {
                        cluster = startTestingCluster(2, 1, DEFAULT_AKKA_ASK_TIMEOUT());
                        final IntermediateDataSetID rid = new IntermediateDataSetID();
                        // Create a task
                        final JobVertex sender = new JobVertex("Sender");
                        sender.setParallelism(1);
                        // just block
                        sender.setInvokableClass(BlockingNoOpInvokable.class);
                        sender.createAndAddResultDataSet(rid, PIPELINED);
                        final JobGraph jobGraph = new JobGraph("Blocking test job", sender);
                        final JobID jid = jobGraph.getJobID();
                        final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
                        // we can set the leader session ID to None because we don't use this gateway to send messages
                        final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
                        // Submit the job and wait for all vertices to be running
                        jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
                        expectMsgClass(JobSubmitSuccess.class);
                        jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
                        expectMsgClass(AllVerticesRunning.class);
                        // This is the mock execution ID of the task requesting the state of the partition
                        final ExecutionAttemptID receiver = new ExecutionAttemptID();
                        // Request the execution graph to get the runtime info
                        jobManagerGateway.tell(new RequestExecutionGraph(jid), testActorGateway);
                        final ExecutionGraph eg = (ExecutionGraph) expectMsgClass(ExecutionGraphFound.class).executionGraph();
                        final ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
                        final IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
                        final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
                        // - The test ----------------------------------------------------------------------
                        // 1. All execution states
                        RequestPartitionProducerState request = new RequestPartitionProducerState(jid, rid, partitionId);
                        for (ExecutionState state : ExecutionState.values()) {
                            ExecutionGraphTestUtils.setVertexState(vertex, state);
                            Future<ExecutionState> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime()).mapTo(ClassTag$.MODULE$.<ExecutionState>apply(ExecutionState.class));
                            ExecutionState resp = Await.result(futurePartitionState, getRemainingTime());
                            assertEquals(state, resp);
                        }
                        // 2. Non-existing execution
                        request = new RequestPartitionProducerState(jid, rid, new ResultPartitionID());
                        Future<?> futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
                        try {
                            Await.result(futurePartitionState, getRemainingTime());
                            fail("Did not fail with expected RuntimeException");
                        } catch (RuntimeException e) {
                            assertEquals(IllegalArgumentException.class, e.getCause().getClass());
                        }
                        // 3. Non-existing job
                        request = new RequestPartitionProducerState(new JobID(), rid, new ResultPartitionID());
                        futurePartitionState = jobManagerGateway.ask(request, getRemainingTime());
                        try {
                            Await.result(futurePartitionState, getRemainingTime());
                            fail("Did not fail with expected IllegalArgumentException");
                        } catch (IllegalArgumentException ignored) {
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        fail(e.getMessage());
                    } finally {
                        if (cluster != null) {
                            cluster.shutdown();
                        }
                    }
                }
            };
        }
    };
}

Example 17 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class JobManagerTest method testRequestPartitionStateMoreRecentExecutionAttempt.

/**
	 * Tests the JobManager response when the execution is not registered with
	 * the ExecutionGraph anymore and a new execution attempt is available.
	 */
@Test
public void testRequestPartitionStateMoreRecentExecutionAttempt() throws Exception {
    new JavaTestKit(system) {

        {
            new Within(duration("15 seconds")) {

                @Override
                protected void run() {
                    // Setup
                    TestingCluster cluster = null;
                    try {
                        cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
                        final IntermediateDataSetID rid = new IntermediateDataSetID();
                        // Create a task
                        final JobVertex sender = new JobVertex("Sender");
                        sender.setParallelism(1);
                        // just finish
                        sender.setInvokableClass(NoOpInvokable.class);
                        sender.createAndAddResultDataSet(rid, PIPELINED);
                        final JobVertex sender2 = new JobVertex("Blocking Sender");
                        sender2.setParallelism(1);
                        // just block
                        sender2.setInvokableClass(BlockingNoOpInvokable.class);
                        sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
                        final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
                        final JobID jid = jobGraph.getJobID();
                        final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
                        // we can set the leader session ID to None because we don't use this gateway to send messages
                        final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
                        // Submit the job and wait for all vertices to be running
                        jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
                        expectMsgClass(JobManagerMessages.JobSubmitSuccess.class);
                        jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
                        expectMsgClass(TestingJobManagerMessages.AllVerticesRunning.class);
                        Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
                        ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
                        ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
                        ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
                        while (vertex.getExecutionState() != ExecutionState.FINISHED) {
                            Thread.sleep(1);
                        }
                        IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
                        ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
                        // Reset execution => new execution attempt
                        vertex.resetForNewExecution();
                        // Producer finished, request state
                        Object request = new JobManagerMessages.RequestPartitionProducerState(jid, rid, partitionId);
                        Future<?> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime());
                        try {
                            Await.result(producerStateFuture, getRemainingTime());
                            fail("Did not fail with expected Exception");
                        } catch (PartitionProducerDisposedException ignored) {
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        fail(e.getMessage());
                    } finally {
                        if (cluster != null) {
                            cluster.shutdown();
                        }
                    }
                }
            };
        }
    };
}

Example 18 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class SingleInputGateTest method testUpdateChannelBeforeRequest.

/**
	 * Tests that an update channel does not trigger a partition request before the UDF has
	 * requested any partitions. Otherwise, this can lead to races when registering a listener at
	 * the gate (e.g. in UnionInputGate), which can result in missed buffer notifications at the
	 * listener.
	 */
@Test
public void testUpdateChannelBeforeRequest() throws Exception {
    SingleInputGate inputGate = new SingleInputGate("t1", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 1, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    InputChannel unknown = new UnknownInputChannel(inputGate, 0, new ResultPartitionID(), partitionManager, new TaskEventDispatcher(), new LocalConnectionManager(), 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    inputGate.setInputChannel(unknown.partitionId.getPartitionId(), unknown);
    // Update to a local channel and verify that no request is triggered
    inputGate.updateInputChannel(new InputChannelDeploymentDescriptor(unknown.partitionId, ResultPartitionLocation.createLocal()));
    verify(partitionManager, never()).createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class));
}

Example 19 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class UnionInputGateTest method testBasicGetNextLogic.

/**
	 * Tests basic correctness of buffer-or-event interleaving and correct <code>null</code> return
	 * value after receiving all end-of-partition events.
	 *
	 * <p> For buffer-or-event instances, it is important to verify that they have been set off to
	 * the correct logical index.
	 */
@Test(timeout = 120 * 1000)
public void testBasicGetNextLogic() throws Exception {
    // Setup
    final String testTaskName = "Test Task";
    final SingleInputGate ig1 = new SingleInputGate(testTaskName, new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 3, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final SingleInputGate ig2 = new SingleInputGate(testTaskName, new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 5, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final UnionInputGate union = new UnionInputGate(new SingleInputGate[] { ig1, ig2 });
    assertEquals(ig1.getNumberOfInputChannels() + ig2.getNumberOfInputChannels(), union.getNumberOfInputChannels());
    final TestInputChannel[][] inputChannels = new TestInputChannel[][] { TestInputChannel.createInputChannels(ig1, 3), TestInputChannel.createInputChannels(ig2, 5) };
    // 0 => 0
    inputChannels[0][0].readBuffer();
    // 0 => 0
    inputChannels[0][0].readEndOfPartitionEvent();
    // 2 => 5
    inputChannels[1][2].readBuffer();
    // 2 => 5
    inputChannels[1][2].readEndOfPartitionEvent();
    // 0 => 3
    inputChannels[1][0].readBuffer();
    // 1 => 4
    inputChannels[1][1].readBuffer();
    // 1 => 1
    inputChannels[0][1].readBuffer();
    // 3 => 6
    inputChannels[1][3].readBuffer();
    // 1 => 1
    inputChannels[0][1].readEndOfPartitionEvent();
    // 3 => 6
    inputChannels[1][3].readEndOfPartitionEvent();
    // 1 => 2
    inputChannels[0][2].readBuffer();
    // 1 => 2
    inputChannels[0][2].readEndOfPartitionEvent();
    // 4 => 7
    inputChannels[1][4].readBuffer();
    // 4 => 7
    inputChannels[1][4].readEndOfPartitionEvent();
    // 0 => 3
    inputChannels[1][1].readEndOfPartitionEvent();
    // 0 => 3
    inputChannels[1][0].readEndOfPartitionEvent();
    ig1.notifyChannelNonEmpty(inputChannels[0][0].getInputChannel());
    ig1.notifyChannelNonEmpty(inputChannels[0][1].getInputChannel());
    ig1.notifyChannelNonEmpty(inputChannels[0][2].getInputChannel());
    ig2.notifyChannelNonEmpty(inputChannels[1][0].getInputChannel());
    ig2.notifyChannelNonEmpty(inputChannels[1][1].getInputChannel());
    ig2.notifyChannelNonEmpty(inputChannels[1][2].getInputChannel());
    ig2.notifyChannelNonEmpty(inputChannels[1][3].getInputChannel());
    ig2.notifyChannelNonEmpty(inputChannels[1][4].getInputChannel());
    // gate 1, channel 0
    SingleInputGateTest.verifyBufferOrEvent(union, true, 0);
    // gate 2, channel 0
    SingleInputGateTest.verifyBufferOrEvent(union, true, 3);
    // gate 1, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, true, 1);
    // gate 2, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, true, 4);
    // gate 1, channel 2
    SingleInputGateTest.verifyBufferOrEvent(union, true, 2);
    // gate 2, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, true, 5);
    // gate 1, channel 0
    SingleInputGateTest.verifyBufferOrEvent(union, false, 0);
    // gate 2, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, true, 6);
    // gate 1, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, false, 1);
    // gate 2, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, true, 7);
    // gate 1, channel 2
    SingleInputGateTest.verifyBufferOrEvent(union, false, 2);
    // gate 2, channel 0
    SingleInputGateTest.verifyBufferOrEvent(union, false, 3);
    // gate 2, channel 1
    SingleInputGateTest.verifyBufferOrEvent(union, false, 4);
    // gate 2, channel 2
    SingleInputGateTest.verifyBufferOrEvent(union, false, 5);
    // gate 2, channel 3
    SingleInputGateTest.verifyBufferOrEvent(union, false, 6);
    // gate 2, channel 4
    SingleInputGateTest.verifyBufferOrEvent(union, false, 7);
    // Return null when the input gate has received all end-of-partition events
    assertTrue(union.isFinished());
    assertNull(union.getNextBufferOrEvent());
}

Example 20 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class ExecutionVertex method createDeploymentDescriptor.

/**
	 * Creates a task deployment descriptor to deploy a subtask to the given target slot.
	 *
	 * TODO: This should actually be in the EXECUTION
	 */
TaskDeploymentDescriptor createDeploymentDescriptor(ExecutionAttemptID executionId, SimpleSlot targetSlot, TaskStateHandles taskStateHandles, int attemptNumber) throws ExecutionGraphException {
    // Produced intermediate results
    List<ResultPartitionDeploymentDescriptor> producedPartitions = new ArrayList<>(resultPartitions.size());
    // Consumed intermediate results
    List<InputGateDeploymentDescriptor> consumedPartitions = new ArrayList<>(inputEdges.length);
    boolean lazyScheduling = getExecutionGraph().getScheduleMode().allowLazyDeployment();
    for (IntermediateResultPartition partition : resultPartitions.values()) {
        List<List<ExecutionEdge>> consumers = partition.getConsumers();
        if (consumers.isEmpty()) {
            //TODO this case only exists for test, currently there has to be exactly one consumer in real jobs!
            producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, ExecutionConfig.UPPER_BOUND_MAX_PARALLELISM, lazyScheduling));
        } else {
            Preconditions.checkState(1 == consumers.size(), "Only one consumer supported in the current implementation! Found: " + consumers.size());
            List<ExecutionEdge> consumer = consumers.get(0);
            ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex();
            int maxParallelism = vertex.getMaxParallelism();
            producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling));
        }
    }
    for (ExecutionEdge[] edges : inputEdges) {
        InputChannelDeploymentDescriptor[] partitions = InputChannelDeploymentDescriptor.fromEdges(edges, targetSlot, lazyScheduling);
        // If the produced partition has multiple consumers registered, we
        // need to request the one matching our sub task index.
        // TODO Refactor after removing the consumers from the intermediate result partitions
        int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size();
        int queueToRequest = subTaskIndex % numConsumerEdges;
        IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult();
        final IntermediateDataSetID resultId = consumedIntermediateResult.getId();
        final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
        consumedPartitions.add(new InputGateDeploymentDescriptor(resultId, partitionType, queueToRequest, partitions));
    }
    SerializedValue<JobInformation> serializedJobInformation = getExecutionGraph().getSerializedJobInformation();
    SerializedValue<TaskInformation> serializedJobVertexInformation = null;
    try {
        serializedJobVertexInformation = jobVertex.getSerializedTaskInformation();
    } catch (IOException e) {
        throw new ExecutionGraphException("Could not create a serialized JobVertexInformation for " + jobVertex.getJobVertexId(), e);
    }
    return new TaskDeploymentDescriptor(serializedJobInformation, serializedJobVertexInformation, executionId, targetSlot.getAllocatedSlot().getSlotAllocationId(), subTaskIndex, attemptNumber, targetSlot.getRoot().getSlotNumber(), taskStateHandles, producedPartitions, consumedPartitions);
}