Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 56 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmitTest method testFailureWhenJarBlobsMissing.

@Test
public void testFailureWhenJarBlobsMissing() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Test Vertex");
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // request the blob port from the job manager
        Future<Object> future = jmGateway.ask(JobManagerMessages.getRequestBlobManagerPort(), timeout);
        int blobPort = (Integer) Await.result(future, timeout);
        // upload two dummy bytes and add their keys to the job graph as dependencies
        BlobKey key1, key2;
        BlobClient bc = new BlobClient(new InetSocketAddress("localhost", blobPort), jmConfig);
        try {
            key1 = bc.put(new byte[10]);
            key2 = bc.put(new byte[10]);
            // delete one of the blobs to make sure that the startup failed
            bc.delete(key2);
        } finally {
            bc.close();
        }
        jg.addBlob(key1);
        jg.addBlob(key2);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            assertTrue(e.getCause() instanceof IOException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmitTest method testFailureWhenInitializeOnMasterFails.

/**
	 * Verifies a correct error message when vertices with master initialization
	 * (input formats / output formats) fail.
	 */
@Test
public void testFailureWhenInitializeOnMasterFails() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Vertex that fails in initializeOnMaster") {

            private static final long serialVersionUID = -3540303593784587652L;

            @Override
            public void initializeOnMaster(ClassLoader loader) throws Exception {
                throw new RuntimeException("test exception");
            }
        };
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            // test that the exception nesting is not too deep
            assertTrue(e.getCause() instanceof RuntimeException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 58 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}

Example 59 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmitTest method testAnswerFailureWhenSavepointReadFails.

@Test
public void testAnswerFailureWhenSavepointReadFails() throws Exception {
    // create a simple job graph
    JobGraph jg = createSimpleJobGraph();
    jg.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("pathThatReallyDoesNotExist..."));
    // submit the job
    Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.DETACHED), timeout);
    Object result = Await.result(submitFuture, timeout);
    assertEquals(JobManagerMessages.JobResultFailure.class, result.getClass());
}

Example 60 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class SlotCountExceedingParallelismTest method createTestJobGraph.

private JobGraph createTestJobGraph(String jobName, int senderParallelism, int receiverParallelism) {
    // The sender and receiver invokable logic ensure that each subtask gets the expected data
    final JobVertex sender = new JobVertex("Sender");
    sender.setInvokableClass(RoundRobinSubtaskIndexSender.class);
    sender.getConfiguration().setInteger(RoundRobinSubtaskIndexSender.CONFIG_KEY, receiverParallelism);
    sender.setParallelism(senderParallelism);
    final JobVertex receiver = new JobVertex("Receiver");
    receiver.setInvokableClass(SubtaskIndexReceiver.class);
    receiver.getConfiguration().setInteger(SubtaskIndexReceiver.CONFIG_KEY, senderParallelism);
    receiver.setParallelism(receiverParallelism);
    receiver.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
    final JobGraph jobGraph = new JobGraph(jobName, sender, receiver);
    // We need to allow queued scheduling, because there are not enough slots available
    // to run all tasks at once. We queue tasks and then let them finish/consume the blocking
    // result one after the other.
    jobGraph.setAllowQueuedScheduling(true);
    return jobGraph;
}