Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 36 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobManagerTest method testRequestPartitionStateMoreRecentExecutionAttempt.

/**
	 * Tests the JobManager response when the execution is not registered with
	 * the ExecutionGraph anymore and a new execution attempt is available.
	 */
@Test
public void testRequestPartitionStateMoreRecentExecutionAttempt() throws Exception {
    new JavaTestKit(system) {

        {
            new Within(duration("15 seconds")) {

                @Override
                protected void run() {
                    // Setup
                    TestingCluster cluster = null;
                    try {
                        cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
                        final IntermediateDataSetID rid = new IntermediateDataSetID();
                        // Create a task
                        final JobVertex sender = new JobVertex("Sender");
                        sender.setParallelism(1);
                        // just finish
                        sender.setInvokableClass(NoOpInvokable.class);
                        sender.createAndAddResultDataSet(rid, PIPELINED);
                        final JobVertex sender2 = new JobVertex("Blocking Sender");
                        sender2.setParallelism(1);
                        // just block
                        sender2.setInvokableClass(BlockingNoOpInvokable.class);
                        sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
                        final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
                        final JobID jid = jobGraph.getJobID();
                        final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
                        // we can set the leader session ID to None because we don't use this gateway to send messages
                        final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
                        // Submit the job and wait for all vertices to be running
                        jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
                        expectMsgClass(JobManagerMessages.JobSubmitSuccess.class);
                        jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
                        expectMsgClass(TestingJobManagerMessages.AllVerticesRunning.class);
                        Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
                        ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
                        ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
                        ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
                        while (vertex.getExecutionState() != ExecutionState.FINISHED) {
                            Thread.sleep(1);
                        }
                        IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
                        ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
                        // Reset execution => new execution attempt
                        vertex.resetForNewExecution();
                        // Producer finished, request state
                        Object request = new JobManagerMessages.RequestPartitionProducerState(jid, rid, partitionId);
                        Future<?> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime());
                        try {
                            Await.result(producerStateFuture, getRemainingTime());
                            fail("Did not fail with expected Exception");
                        } catch (PartitionProducerDisposedException ignored) {
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                        fail(e.getMessage());
                    } finally {
                        if (cluster != null) {
                            cluster.shutdown();
                        }
                    }
                }
            };
        }
    };
}

Example 37 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobSubmitTest method testFailureWhenJarBlobsMissing.

@Test
public void testFailureWhenJarBlobsMissing() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Test Vertex");
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // request the blob port from the job manager
        Future<Object> future = jmGateway.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        int blobPort = (Integer) Await.result(future, timeout);
        // upload two dummy bytes and add their keys to the job graph as dependencies
        BlobKey key1, key2;
        BlobClient bc = new BlobClient(new InetSocketAddress("localhost", blobPort), jmConfig);
        try {
            key1 = bc.put(new byte[10]);
            key2 = bc.put(new byte[10]);
            // delete one of the blobs to make sure that the startup failed
            bc.delete(key2);
        } finally {
            bc.close();
        }
        jg.addBlob(key1);
        jg.addBlob(key2);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            assertTrue(e.getCause() instanceof IOException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 38 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobSubmitTest method testFailureWhenInitializeOnMasterFails.

/**
	 * Verifies a correct error message when vertices with master initialization
	 * (input formats / output formats) fail.
	 */
@Test
public void testFailureWhenInitializeOnMasterFails() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Vertex that fails in initializeOnMaster") {

            private static final long serialVersionUID = -3540303593784587652L;

            @Override
            public void initializeOnMaster(ClassLoader loader) throws Exception {
                throw new RuntimeException("test exception");
            }
        };
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            // test that the exception nesting is not too deep
            assertTrue(e.getCause() instanceof RuntimeException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 39 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}

Example 40 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class CoLocationConstraintTest method testAssignSlotAndLockLocation.

@Test
public void testAssignSlotAndLockLocation() {
    try {
        JobID jid = new JobID();
        JobVertex vertex = new JobVertex("vertex");
        vertex.setParallelism(1);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vertex.getID());
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        CoLocationGroup constraintGroup = new CoLocationGroup(vertex);
        CoLocationConstraint constraint = constraintGroup.getLocationConstraint(0);
        // constraint is completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        Instance instance1 = SchedulerTestUtils.getRandomInstance(2);
        Instance instance2 = SchedulerTestUtils.getRandomInstance(2);
        SharedSlot slot1_1 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot1_2 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_1 = instance2.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_2 = instance2.allocateSharedSlot(jid, assignment);
        // constraint is still completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // set the slot, but do not lock the location yet
        constraint.setSharedSlot(slot1_1);
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // try to get the location
        try {
            constraint.getLocation();
            fail("should throw an IllegalStateException");
        } catch (IllegalStateException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalStateException");
        }
        // check that we can reassign the slot as long as the location is not locked
        constraint.setSharedSlot(slot2_1);
        // the previous slot should have been released now
        assertTrue(slot1_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // we can do an identity re-assign
        constraint.setSharedSlot(slot2_1);
        assertFalse(slot2_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        constraint.lockLocation();
        // now, the location is assigned and we have a location
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // release the slot
        slot2_1.releaseSlot();
        // we should still have a location
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // we can not assign a different location
        try {
            constraint.setSharedSlot(slot1_2);
            fail("should throw an IllegalArgumentException");
        } catch (IllegalArgumentException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalArgumentException");
        }
        // assign a new slot with the same location
        constraint.setSharedSlot(slot2_2);
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}