Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 16 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class SharedSlotsTest method testAllocateAndReleaseTwoLevels.

/**
	 * We allocate and release the structure below, starting by allocating a simple slot in the
	 * shared slot and finishing by releasing a simple slot.
	 * 
	 * <pre>
	 *     Shared(0)(root)
	 *        |
	 *        +-- Simple(2)(sink)
	 *        |
	 *        +-- Shared(1)(co-location-group)
	 *        |      |
	 *        |      +-- Simple(0)(tail)
	 *        |      +-- Simple(1)(head)
	 *        |
	 *        +-- Simple(0)(source)
	 * </pre>
	 */
@Test
public void testAllocateAndReleaseTwoLevels() {
    try {
        JobVertexID sourceId = new JobVertexID();
        JobVertexID headId = new JobVertexID();
        JobVertexID tailId = new JobVertexID();
        JobVertexID sinkId = new JobVertexID();
        JobVertex headVertex = new JobVertex("head", headId);
        JobVertex tailVertex = new JobVertex("tail", tailId);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        assertEquals(0, assignment.getNumberOfSlots());
        CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
        CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
        assertFalse(constraint.isAssigned());
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
        // get the first simple slot
        SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
        assertEquals(1, sharedSlot.getNumberLeaves());
        // get the first slot in the nested shared slot from the co-location constraint
        SimpleSlot headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isAlive());
        assertFalse(constraint.isAssigned());
        // we do not immediately lock the location
        headSlot.releaseSlot();
        assertEquals(1, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertFalse(constraint.isAssigned());
        // re-allocate the head slot
        headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        constraint.lockLocation();
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        SimpleSlot tailSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(constraint.getSharedSlot(), headSlot.getParent());
        assertEquals(constraint.getSharedSlot(), tailSlot.getParent());
        SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, Collections.<TaskManagerLocation>emptySet());
        assertEquals(4, sharedSlot.getNumberLeaves());
        // we release our co-location constraint tasks
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        // we should have resources again for the co-location constraint
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // re-allocate head and tail from the constraint
        headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        assertEquals(4, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // verify some basic properties of the slots
        assertEquals(instance.getTaskManagerID(), sourceSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), headSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), tailSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), sinkSlot.getTaskManagerID());
        assertEquals(sourceId, sourceSlot.getGroupID());
        assertEquals(sinkId, sinkSlot.getGroupID());
        assertNull(headSlot.getGroupID());
        assertNull(tailSlot.getGroupID());
        assertEquals(constraint.getGroupId(), constraint.getSharedSlot().getGroupID());
        // release all
        sourceSlot.releaseSlot();
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        sinkSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sourceSlot.isReleased());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(sinkSlot.isReleased());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 17 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobSubmissionFailsITCase method setup.

@BeforeClass
public static void setup() {
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
        config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS / 2);
        cluster = new LocalFlinkMiniCluster(config);
        cluster.start();
        final JobVertex jobVertex = new JobVertex("Working job vertex.");
        jobVertex.setInvokableClass(NoOpInvokable.class);
        workingJobGraph = new JobGraph("Working testing job", jobVertex);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 18 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobSubmissionFailsITCase method testExceptionInInitializeOnMaster.

@Test
public void testExceptionInInitializeOnMaster() {
    try {
        final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex");
        failingJobVertex.setInvokableClass(NoOpInvokable.class);
        final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex);
        try {
            submitJob(failingJobGraph);
            fail("Expected JobExecutionException.");
        } catch (JobExecutionException e) {
            assertEquals("Test exception.", e.getCause().getMessage());
        } catch (Throwable t) {
            t.printStackTrace();
            fail("Caught wrong exception of type " + t.getClass() + ".");
        }
        cluster.submitJobAndWait(workingJobGraph, false);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 19 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphConstructionTest method testCreateSimpleGraphBipartite.

/**
	 * Creates a JobGraph of the following form:
	 * 
	 * <pre>
	 *  v1--->v2-->\
	 *              \
	 *               v4 --->\
	 *        ----->/        \
	 *  v3-->/                v5
	 *       \               /
	 *        ------------->/
	 * </pre>
	 */
@Test
public void testCreateSimpleGraphBipartite() throws Exception {
    final JobID jobId = new JobID();
    final String jobName = "Test Job Sample Name";
    final Configuration cfg = new Configuration();
    JobVertex v1 = new JobVertex("vertex1");
    JobVertex v2 = new JobVertex("vertex2");
    JobVertex v3 = new JobVertex("vertex3");
    JobVertex v4 = new JobVertex("vertex4");
    JobVertex v5 = new JobVertex("vertex5");
    v1.setParallelism(5);
    v2.setParallelism(7);
    v3.setParallelism(2);
    v4.setParallelism(11);
    v5.setParallelism(4);
    v1.setInvokableClass(AbstractInvokable.class);
    v2.setInvokableClass(AbstractInvokable.class);
    v3.setInvokableClass(AbstractInvokable.class);
    v4.setInvokableClass(AbstractInvokable.class);
    v5.setInvokableClass(AbstractInvokable.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
    ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
    try {
        eg.attachJobGraph(ordered);
    } catch (JobException e) {
        e.printStackTrace();
        fail("Job failed with exception: " + e.getMessage());
    }
    verifyTestGraph(eg, jobId, v1, v2, v3, v4, v5);
}

Example 20 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphConstructionTest method testMoreThanOneConsumerForIntermediateResult.

@Test
public void testMoreThanOneConsumerForIntermediateResult() {
    try {
        final JobID jobId = new JobID();
        final String jobName = "Test Job Sample Name";
        final Configuration cfg = new Configuration();
        JobVertex v1 = new JobVertex("vertex1");
        JobVertex v2 = new JobVertex("vertex2");
        JobVertex v3 = new JobVertex("vertex3");
        v1.setParallelism(5);
        v2.setParallelism(7);
        v3.setParallelism(2);
        IntermediateDataSet result = v1.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
        v2.connectDataSetAsInput(result, DistributionPattern.ALL_TO_ALL);
        v3.connectDataSetAsInput(result, DistributionPattern.ALL_TO_ALL);
        List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3));
        ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        try {
            eg.attachJobGraph(ordered);
            fail("Should not be possible");
        } catch (RuntimeException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}