Examples with SlotSharingGroup org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup used on opensource projects

Example 31 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SharedSlotsTest method allocateSimpleSlotsAndReleaseFromRoot.

@Test
public void allocateSimpleSlotsAndReleaseFromRoot() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        JobVertexID vid4 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3, vid4);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, vid1);
        assertNotNull(sub1);
        assertNull(sub1.getExecutedVertex());
        assertEquals(Locality.LOCAL, sub1.getLocality());
        assertEquals(1, sub1.getNumberLeaves());
        assertEquals(vid1, sub1.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub1.getTaskManagerID());
        assertEquals(jobId, sub1.getJobID());
        assertEquals(sharedSlot, sub1.getParent());
        assertEquals(sharedSlot, sub1.getRoot());
        assertEquals(0, sub1.getRootSlotNumber());
        assertEquals(0, sub1.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        assertNotNull(sub2);
        assertNull(sub2.getExecutedVertex());
        assertEquals(Locality.UNCONSTRAINED, sub2.getLocality());
        assertEquals(1, sub2.getNumberLeaves());
        assertEquals(vid2, sub2.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub2.getTaskManagerID());
        assertEquals(jobId, sub2.getJobID());
        assertEquals(sharedSlot, sub2.getParent());
        assertEquals(sharedSlot, sub2.getRoot());
        assertEquals(0, sub2.getRootSlotNumber());
        assertEquals(1, sub2.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, Collections.singleton(instance.getTaskManagerLocation()));
        assertNotNull(sub3);
        assertNull(sub3.getExecutedVertex());
        assertEquals(Locality.LOCAL, sub3.getLocality());
        assertEquals(1, sub3.getNumberLeaves());
        assertEquals(vid3, sub3.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub3.getTaskManagerID());
        assertEquals(jobId, sub3.getJobID());
        assertEquals(sharedSlot, sub3.getParent());
        assertEquals(sharedSlot, sub3.getRoot());
        assertEquals(0, sub3.getRootSlotNumber());
        assertEquals(2, sub3.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        SimpleSlot sub4 = assignment.getSlotForTask(vid4, Collections.singleton(SchedulerTestUtils.getRandomInstance(1).getTaskManagerLocation()));
        assertNotNull(sub4);
        assertNull(sub4.getExecutedVertex());
        assertEquals(Locality.NON_LOCAL, sub4.getLocality());
        assertEquals(1, sub4.getNumberLeaves());
        assertEquals(vid4, sub4.getGroupID());
        assertEquals(instance.getTaskManagerID(), sub4.getTaskManagerID());
        assertEquals(jobId, sub4.getJobID());
        assertEquals(sharedSlot, sub4.getParent());
        assertEquals(sharedSlot, sub4.getRoot());
        assertEquals(0, sub4.getRootSlotNumber());
        assertEquals(3, sub4.getSlotNumber());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        // release from the root.
        sharedSlot.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isReleased());
        assertTrue(sub4.isReleased());
        assertEquals(0, sharedSlot.getNumberLeaves());
        assertFalse(sharedSlot.hasChildren());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid4));
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 32 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SharedSlotsTest method testImmediateReleaseTwoLevel.

@Test
public void testImmediateReleaseTwoLevel() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid = new JobVertexID();
        JobVertex vertex = new JobVertex("vertex", vid);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        CoLocationGroup coLocationGroup = new CoLocationGroup(vertex);
        CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        SimpleSlot sub = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, constraint);
        assertNull(sub.getGroupID());
        assertEquals(constraint.getSharedSlot(), sub.getParent());
        sub.releaseSlot();
        assertTrue(sub.isReleased());
        assertTrue(sharedSlot.isReleased());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 33 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SharedSlotsTest method testImmediateReleaseOneLevel.

@Test
public void testImmediateReleaseOneLevel() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        SimpleSlot sub = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid);
        sub.releaseSlot();
        assertTrue(sub.isReleased());
        assertTrue(sharedSlot.isReleased());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, instance.getNumberOfAllocatedSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 34 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class LeaderChangeJobRecoveryTest method createBlockingJob.

public JobGraph createBlockingJob(int parallelism) {
    Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true);
    JobVertex sender = new JobVertex("sender");
    JobVertex receiver = new JobVertex("receiver");
    sender.setInvokableClass(Tasks.Sender.class);
    receiver.setInvokableClass(Tasks.BlockingOnceReceiver.class);
    sender.setParallelism(parallelism);
    receiver.setParallelism(parallelism);
    receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    sender.setSlotSharingGroup(slotSharingGroup);
    receiver.setSlotSharingGroup(slotSharingGroup);
    return new JobGraph("Blocking test job", sender, receiver);
}

Example 35 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class LeaderChangeStateCleanupTest method createBlockingJob.

public JobGraph createBlockingJob(int parallelism) {
    Tasks.BlockingOnceReceiver$.MODULE$.blocking_$eq(true);
    JobVertex sender = new JobVertex("sender");
    JobVertex receiver = new JobVertex("receiver");
    sender.setInvokableClass(Tasks.Sender.class);
    receiver.setInvokableClass(Tasks.BlockingOnceReceiver.class);
    sender.setParallelism(parallelism);
    receiver.setParallelism(parallelism);
    receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    sender.setSlotSharingGroup(slotSharingGroup);
    receiver.setSlotSharingGroup(slotSharingGroup);
    return new JobGraph("Blocking test job", sender, receiver);
}