Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 26 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmitTest method testFailureWhenInitializeOnMasterFails.

/**
	 * Verifies a correct error message when vertices with master initialization
	 * (input formats / output formats) fail.
	 */
@Test
public void testFailureWhenInitializeOnMasterFails() {
    try {
        // create a simple job graph
        JobVertex jobVertex = new JobVertex("Vertex that fails in initializeOnMaster") {

            private static final long serialVersionUID = -3540303593784587652L;

            @Override
            public void initializeOnMaster(ClassLoader loader) throws Exception {
                throw new RuntimeException("test exception");
            }
        };
        jobVertex.setInvokableClass(NoOpInvokable.class);
        JobGraph jg = new JobGraph("test job", jobVertex);
        // submit the job
        Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.EXECUTION_RESULT), timeout);
        try {
            Await.result(submitFuture, timeout);
        } catch (JobExecutionException e) {
            // that is what we expect
            // test that the exception nesting is not too deep
            assertTrue(e.getCause() instanceof RuntimeException);
        } catch (Exception e) {
            fail("Wrong exception type");
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 27 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}

Example 28 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmitTest method testAnswerFailureWhenSavepointReadFails.

@Test
public void testAnswerFailureWhenSavepointReadFails() throws Exception {
    // create a simple job graph
    JobGraph jg = createSimpleJobGraph();
    jg.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("pathThatReallyDoesNotExist..."));
    // submit the job
    Future<Object> submitFuture = jmGateway.ask(new JobManagerMessages.SubmitJob(jg, ListeningBehaviour.DETACHED), timeout);
    Object result = Await.result(submitFuture, timeout);
    assertEquals(JobManagerMessages.JobResultFailure.class, result.getClass());
}

Example 29 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class ZooKeeperSubmittedJobGraphsStoreITCase method verifyJobGraphs.

protected void verifyJobGraphs(SubmittedJobGraph expected, SubmittedJobGraph actual) throws Exception {
    JobGraph expectedJobGraph = expected.getJobGraph();
    JobGraph actualJobGraph = actual.getJobGraph();
    assertEquals(expectedJobGraph.getName(), actualJobGraph.getName());
    assertEquals(expectedJobGraph.getJobID(), actualJobGraph.getJobID());
    JobInfo expectedJobInfo = expected.getJobInfo();
    JobInfo actualJobInfo = actual.getJobInfo();
    assertEquals(expectedJobInfo, actualJobInfo);
}

Example 30 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class LocalExecutor method executePlan.

/**
	 * Executes the given program on a local runtime and waits for the job to finish.
	 * 
	 * <p>If the executor has not been started before, this starts the executor and shuts it down
	 * after the job finished. If the job runs in session mode, the executor is kept alive until
	 * no more references to the executor exist.</p>
	 * 
	 * @param plan The plan of the program to execute.
	 * @return The net runtime of the program, in milliseconds.
	 * 
	 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
	 *                   caused an exception.
	 */
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
    if (plan == null) {
        throw new IllegalArgumentException("The plan may not be null.");
    }
    synchronized (this.lock) {
        // check if we start a session dedicated for this execution
        final boolean shutDownAtEnd;
        if (flink == null) {
            shutDownAtEnd = true;
            // configure the number of local slots equal to the parallelism of the local plan
            if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
                int maxParallelism = plan.getMaximumParallelism();
                if (maxParallelism > 0) {
                    this.taskManagerNumSlots = maxParallelism;
                }
            }
            // start the cluster for us
            start();
        } else {
            // we use the existing session
            shutDownAtEnd = false;
        }
        try {
            Configuration configuration = this.flink.configuration();
            Optimizer pc = new Optimizer(new DataStatistics(), configuration);
            OptimizedPlan op = pc.compile(plan);
            JobGraphGenerator jgg = new JobGraphGenerator(configuration);
            JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());
            boolean sysoutPrint = isPrintingStatusDuringExecution();
            return flink.submitJobAndWait(jobGraph, sysoutPrint);
        } finally {
            if (shutDownAtEnd) {
                stop();
            }
        }
    }
}