Examples with CoLocationGroup org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup used on opensource projects

Example 1 with CoLocationGroup

use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.

the class SharedSlotsTest method testReleaseTwoLevelsFromRoot.

/**
	 * We allocate and the structure below and release it from the root.
	 *
	 * <pre>
	 *     Shared(0)(root)
	 *        |
	 *        +-- Simple(2)(sink)
	 *        |
	 *        +-- Shared(1)(co-location-group)
	 *        |      |
	 *        |      +-- Simple(0)(tail)
	 *        |      +-- Simple(1)(head)
	 *        |
	 *        +-- Simple(0)(source)
	 * </pre>
	 */
@Test
public void testReleaseTwoLevelsFromRoot() {
    try {
        JobVertexID sourceId = new JobVertexID();
        JobVertexID headId = new JobVertexID();
        JobVertexID tailId = new JobVertexID();
        JobVertexID sinkId = new JobVertexID();
        JobVertex headVertex = new JobVertex("head", headId);
        JobVertex tailVertex = new JobVertex("tail", tailId);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        assertEquals(0, assignment.getNumberOfSlots());
        CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
        CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
        assertFalse(constraint.isAssigned());
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
        // get the first simple slot
        SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
        SimpleSlot headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        constraint.lockLocation();
        SimpleSlot tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, NO_LOCATION);
        assertEquals(4, sharedSlot.getNumberLeaves());
        // release all
        sourceSlot.releaseSlot();
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        sinkSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sourceSlot.isReleased());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(sinkSlot.isReleased());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
        assertEquals(0, assignment.getNumberOfSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with CoLocationGroup

use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.

the class SharedSlotsTest method testAllocateAndReleaseTwoLevels.

/**
	 * We allocate and release the structure below, starting by allocating a simple slot in the
	 * shared slot and finishing by releasing a simple slot.
	 * 
	 * <pre>
	 *     Shared(0)(root)
	 *        |
	 *        +-- Simple(2)(sink)
	 *        |
	 *        +-- Shared(1)(co-location-group)
	 *        |      |
	 *        |      +-- Simple(0)(tail)
	 *        |      +-- Simple(1)(head)
	 *        |
	 *        +-- Simple(0)(source)
	 * </pre>
	 */
@Test
public void testAllocateAndReleaseTwoLevels() {
    try {
        JobVertexID sourceId = new JobVertexID();
        JobVertexID headId = new JobVertexID();
        JobVertexID tailId = new JobVertexID();
        JobVertexID sinkId = new JobVertexID();
        JobVertex headVertex = new JobVertex("head", headId);
        JobVertex tailVertex = new JobVertex("tail", tailId);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        assertEquals(0, assignment.getNumberOfSlots());
        CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
        CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
        assertFalse(constraint.isAssigned());
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
        // get the first simple slot
        SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
        assertEquals(1, sharedSlot.getNumberLeaves());
        // get the first slot in the nested shared slot from the co-location constraint
        SimpleSlot headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isAlive());
        assertFalse(constraint.isAssigned());
        // we do not immediately lock the location
        headSlot.releaseSlot();
        assertEquals(1, sharedSlot.getNumberLeaves());
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertFalse(constraint.isAssigned());
        // re-allocate the head slot
        headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        constraint.lockLocation();
        assertNotNull(constraint.getSharedSlot());
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        SimpleSlot tailSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
        assertEquals(constraint.getSharedSlot(), headSlot.getParent());
        assertEquals(constraint.getSharedSlot(), tailSlot.getParent());
        SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, Collections.<TaskManagerLocation>emptySet());
        assertEquals(4, sharedSlot.getNumberLeaves());
        // we release our co-location constraint tasks
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
        // we should have resources again for the co-location constraint
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // re-allocate head and tail from the constraint
        headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
        assertEquals(4, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
        // verify some basic properties of the slots
        assertEquals(instance.getTaskManagerID(), sourceSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), headSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), tailSlot.getTaskManagerID());
        assertEquals(instance.getTaskManagerID(), sinkSlot.getTaskManagerID());
        assertEquals(sourceId, sourceSlot.getGroupID());
        assertEquals(sinkId, sinkSlot.getGroupID());
        assertNull(headSlot.getGroupID());
        assertNull(tailSlot.getGroupID());
        assertEquals(constraint.getGroupId(), constraint.getSharedSlot().getGroupID());
        // release all
        sourceSlot.releaseSlot();
        headSlot.releaseSlot();
        tailSlot.releaseSlot();
        sinkSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sourceSlot.isReleased());
        assertTrue(headSlot.isReleased());
        assertTrue(tailSlot.isReleased());
        assertTrue(sinkSlot.isReleased());
        assertTrue(constraint.getSharedSlot().isReleased());
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with CoLocationGroup

use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.

the class LocalInputPreferredSlotSharingStrategyTest method testCoLocationConstraintIsRespected.

@Test
public void testCoLocationConstraintIsRespected() {
    topology.connect(ev11, ev22);
    topology.connect(ev12, ev21);
    final CoLocationGroup coLocationGroup = new TestingCoLocationGroup(JOB_VERTEX_ID_1, JOB_VERTEX_ID_2);
    final Set<CoLocationGroup> coLocationGroups = Collections.singleton(coLocationGroup);
    final SlotSharingStrategy strategy = new LocalInputPreferredSlotSharingStrategy(topology, slotSharingGroups, coLocationGroups);
    assertThat(strategy.getExecutionSlotSharingGroups(), hasSize(2));
    assertThat(strategy.getExecutionSlotSharingGroup(ev11.getId()).getExecutionVertexIds(), containsInAnyOrder(ev11.getId(), ev21.getId()));
    assertThat(strategy.getExecutionSlotSharingGroup(ev12.getId()).getExecutionVertexIds(), containsInAnyOrder(ev12.getId(), ev22.getId()));
}

Example 4 with CoLocationGroup

use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.

the class StreamingJobGraphGeneratorTest method testIteration.

/**
 * Test iteration job, check slot sharing group and co-location group.
 */
@Test
public void testIteration() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 2, 3).name("source");
    IterativeStream<Integer> iteration = source.iterate(3000);
    iteration.name("iteration").setParallelism(2);
    DataStream<Integer> map = iteration.map(x -> x + 1).name("map").setParallelism(2);
    DataStream<Integer> filter = map.filter((x) -> false).name("filter").setParallelism(2);
    iteration.closeWith(filter).print();
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    SlotSharingGroup slotSharingGroup = jobGraph.getVerticesAsArray()[0].getSlotSharingGroup();
    assertNotNull(slotSharingGroup);
    CoLocationGroup iterationSourceCoLocationGroup = null;
    CoLocationGroup iterationSinkCoLocationGroup = null;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        // all vertices have same slot sharing group by default
        assertEquals(slotSharingGroup, jobVertex.getSlotSharingGroup());
        // others have no co-location group by default
        if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SOURCE_NAME_PREFIX)) {
            iterationSourceCoLocationGroup = jobVertex.getCoLocationGroup();
            assertTrue(iterationSourceCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
        } else if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SINK_NAME_PREFIX)) {
            iterationSinkCoLocationGroup = jobVertex.getCoLocationGroup();
            assertTrue(iterationSinkCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
        } else {
            assertNull(jobVertex.getCoLocationGroup());
        }
    }
    assertNotNull(iterationSourceCoLocationGroup);
    assertNotNull(iterationSinkCoLocationGroup);
    assertEquals(iterationSourceCoLocationGroup, iterationSinkCoLocationGroup);
}

Example 5 with CoLocationGroup

use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.

the class JobGraphTest method testGetCoLocationGroups.

@Test
public void testGetCoLocationGroups() {
    final JobVertex v1 = new JobVertex("1");
    final JobVertex v2 = new JobVertex("2");
    final JobVertex v3 = new JobVertex("3");
    final JobVertex v4 = new JobVertex("4");
    final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    v1.setSlotSharingGroup(slotSharingGroup);
    v2.setSlotSharingGroup(slotSharingGroup);
    v1.setStrictlyCoLocatedWith(v2);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(Arrays.asList(v1, v2, v3, v4)).build();
    assertThat(jobGraph.getCoLocationGroups(), hasSize(1));
    final CoLocationGroup onlyCoLocationGroup = jobGraph.getCoLocationGroups().iterator().next();
    assertThat(onlyCoLocationGroup.getVertexIds(), containsInAnyOrder(v1.getID(), v2.getID()));
}