Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 11 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmissionFailsITCase method testSubmitEmptyJobGraph.

@Test
public void testSubmitEmptyJobGraph() {
    try {
        final JobGraph jobGraph = new JobGraph("Testing job");
        try {
            submitJob(jobGraph);
            fail("Expected JobSubmissionException.");
        } catch (JobSubmissionException e) {
            assertTrue(e.getMessage() != null && e.getMessage().contains("empty"));
        } catch (Throwable t) {
            t.printStackTrace();
            fail("Caught wrong exception of type " + t.getClass() + ".");
        }
        cluster.submitJobAndWait(workingJobGraph, false);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 12 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmissionFailsITCase method setup.

@BeforeClass
public static void setup() {
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
        config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS / 2);
        cluster = new LocalFlinkMiniCluster(config);
        cluster.start();
        final JobVertex jobVertex = new JobVertex("Working job vertex.");
        jobVertex.setInvokableClass(NoOpInvokable.class);
        workingJobGraph = new JobGraph("Working testing job", jobVertex);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 13 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobSubmissionFailsITCase method testExceptionInInitializeOnMaster.

@Test
public void testExceptionInInitializeOnMaster() {
    try {
        final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex");
        failingJobVertex.setInvokableClass(NoOpInvokable.class);
        final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex);
        try {
            submitJob(failingJobGraph);
            fail("Expected JobExecutionException.");
        } catch (JobExecutionException e) {
            assertEquals("Test exception.", e.getCause().getMessage());
        } catch (Throwable t) {
            t.printStackTrace();
            fail("Caught wrong exception of type " + t.getClass() + ".");
        }
        cluster.submitJobAndWait(workingJobGraph, false);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class ExecutionGraphConstructionTest method testCoLocationConstraintCreation.

@Test
public void testCoLocationConstraintCreation() {
    try {
        final JobID jobId = new JobID();
        final String jobName = "Co-Location Constraint Sample Job";
        final Configuration cfg = new Configuration();
        // simple group of two, cyclic
        JobVertex v1 = new JobVertex("vertex1");
        JobVertex v2 = new JobVertex("vertex2");
        v1.setParallelism(6);
        v2.setParallelism(4);
        v1.setInvokableClass(AbstractInvokable.class);
        v2.setInvokableClass(AbstractInvokable.class);
        SlotSharingGroup sl1 = new SlotSharingGroup();
        v1.setSlotSharingGroup(sl1);
        v2.setSlotSharingGroup(sl1);
        v2.setStrictlyCoLocatedWith(v1);
        v1.setStrictlyCoLocatedWith(v2);
        // complex forked dependency pattern
        JobVertex v3 = new JobVertex("vertex3");
        JobVertex v4 = new JobVertex("vertex4");
        JobVertex v5 = new JobVertex("vertex5");
        JobVertex v6 = new JobVertex("vertex6");
        JobVertex v7 = new JobVertex("vertex7");
        v3.setParallelism(3);
        v4.setParallelism(3);
        v5.setParallelism(3);
        v6.setParallelism(3);
        v7.setParallelism(3);
        v3.setInvokableClass(AbstractInvokable.class);
        v4.setInvokableClass(AbstractInvokable.class);
        v5.setInvokableClass(AbstractInvokable.class);
        v6.setInvokableClass(AbstractInvokable.class);
        v7.setInvokableClass(AbstractInvokable.class);
        SlotSharingGroup sl2 = new SlotSharingGroup();
        v3.setSlotSharingGroup(sl2);
        v4.setSlotSharingGroup(sl2);
        v5.setSlotSharingGroup(sl2);
        v6.setSlotSharingGroup(sl2);
        v7.setSlotSharingGroup(sl2);
        v4.setStrictlyCoLocatedWith(v3);
        v5.setStrictlyCoLocatedWith(v4);
        v6.setStrictlyCoLocatedWith(v3);
        v3.setStrictlyCoLocatedWith(v7);
        // isolated vertex
        JobVertex v8 = new JobVertex("vertex8");
        v8.setParallelism(2);
        v8.setInvokableClass(AbstractInvokable.class);
        JobGraph jg = new JobGraph(jobId, jobName, v1, v2, v3, v4, v5, v6, v7, v8);
        ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        eg.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
        // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
        {
            ExecutionVertex[] v1s = eg.getJobVertex(v1.getID()).getTaskVertices();
            ExecutionVertex[] v2s = eg.getJobVertex(v2.getID()).getTaskVertices();
            Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
            for (int i = 0; i < v2.getParallelism(); i++) {
                assertNotNull(v1s[i].getLocationConstraint());
                assertNotNull(v2s[i].getLocationConstraint());
                assertTrue(v1s[i].getLocationConstraint() == v2s[i].getLocationConstraint());
                all.add(v1s[i].getLocationConstraint());
            }
            for (int i = v2.getParallelism(); i < v1.getParallelism(); i++) {
                assertNotNull(v1s[i].getLocationConstraint());
                all.add(v1s[i].getLocationConstraint());
            }
            assertEquals("not all co location constraints are distinct", v1.getParallelism(), all.size());
        }
        // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
        {
            ExecutionVertex[] v3s = eg.getJobVertex(v3.getID()).getTaskVertices();
            ExecutionVertex[] v4s = eg.getJobVertex(v4.getID()).getTaskVertices();
            ExecutionVertex[] v5s = eg.getJobVertex(v5.getID()).getTaskVertices();
            ExecutionVertex[] v6s = eg.getJobVertex(v6.getID()).getTaskVertices();
            ExecutionVertex[] v7s = eg.getJobVertex(v7.getID()).getTaskVertices();
            Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
            for (int i = 0; i < v3.getParallelism(); i++) {
                assertNotNull(v3s[i].getLocationConstraint());
                assertTrue(v3s[i].getLocationConstraint() == v4s[i].getLocationConstraint());
                assertTrue(v4s[i].getLocationConstraint() == v5s[i].getLocationConstraint());
                assertTrue(v5s[i].getLocationConstraint() == v6s[i].getLocationConstraint());
                assertTrue(v6s[i].getLocationConstraint() == v7s[i].getLocationConstraint());
                all.add(v3s[i].getLocationConstraint());
            }
            assertEquals("not all co location constraints are distinct", v3.getParallelism(), all.size());
        }
        // check the v8 has no co location hints
        {
            ExecutionVertex[] v8s = eg.getJobVertex(v8.getID()).getTaskVertices();
            for (int i = 0; i < v8.getParallelism(); i++) {
                assertNull(v8s[i].getLocationConstraint());
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 15 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class CoordinatorShutdownTest method testCoordinatorShutsDownOnSuccess.

@Test
public void testCoordinatorShutsDownOnSuccess() {
    LocalFlinkMiniCluster cluster = null;
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
        cluster = new LocalFlinkMiniCluster(config, true);
        cluster.start();
        // build a test graph with snapshotting enabled
        JobVertex vertex = new JobVertex("Test Vertex");
        vertex.setInvokableClass(BlockingInvokable.class);
        List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
        JobGraph testGraph = new JobGraph("test job", vertex);
        testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), null, true));
        ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
        FiniteDuration timeout = new FiniteDuration(60, TimeUnit.SECONDS);
        JobManagerMessages.SubmitJob submitMessage = new JobManagerMessages.SubmitJob(testGraph, ListeningBehaviour.EXECUTION_RESULT);
        // submit is successful, but then the job blocks due to the invokable
        Future<Object> submitFuture = jmGateway.ask(submitMessage, timeout);
        Await.result(submitFuture, timeout);
        // get the execution graph and store the ExecutionGraph reference
        Future<Object> jobRequestFuture = jmGateway.ask(new JobManagerMessages.RequestJob(testGraph.getJobID()), timeout);
        ExecutionGraph graph = (ExecutionGraph) ((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
        assertNotNull(graph);
        BlockingInvokable.unblock();
        graph.waitUntilFinished();
        // verify that the coordinator was shut down
        CheckpointCoordinator coord = graph.getCheckpointCoordinator();
        assertTrue(coord == null || coord.isShutdown());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
            cluster.awaitTermination();
        }
    }
}