Examples with SimpleSlot org.apache.flink.runtime.instance.SimpleSlot used on opensource projects

Example 66 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetweenAndNoNewLocal.

@Test
public void testSlotReleasedInBetweenAndNoNewLocal() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jidx = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        s1.releaseSlot();
        s2.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        SimpleSlot sa = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2)), false).get();
        SimpleSlot sb = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2)), false).get();
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
            fail("should not be able to find a resource");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("wrong exception");
        }
        sa.releaseSlot();
        sb.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(2, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 67 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class ScheduleWithCoLocationHintTest method testGetsNonLocalFromSharingGroupFirst.

@Test
public void testGetsNonLocalFromSharingGroupFirst() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        // schedule something into the shared group so that both instances are in the sharing group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup), false).get();
        // schedule one locally to instance 1
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup, cc1), false).get();
        // schedule with co location constraint (yet unassigned) and a preference for
        // instance 1, but it can only get instance 2
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
        // schedule something into the assigned co-location constraints and check that they override the
        // other preferences
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, loc2), sharingGroup, cc1), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, loc1), sharingGroup, cc2), false).get();
        // check that each slot got three
        assertEquals(3, s1.getRoot().getNumberLeaves());
        assertEquals(3, s2.getRoot().getNumberLeaves());
        assertEquals(s1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(s1.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
        // release some slots, be sure that new available ones come up
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 68 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleAllSharedAndCoLocated.

@Test
public void scheduleAllSharedAndCoLocated() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        assertEquals(6, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint c1 = new CoLocationConstraint(ccg);
        CoLocationConstraint c2 = new CoLocationConstraint(ccg);
        CoLocationConstraint c3 = new CoLocationConstraint(ccg);
        CoLocationConstraint c4 = new CoLocationConstraint(ccg);
        CoLocationConstraint c5 = new CoLocationConstraint(ccg);
        CoLocationConstraint c6 = new CoLocationConstraint(ccg);
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 6), sharingGroup, c1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 6), sharingGroup, c2), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 6), sharingGroup, c3), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 6), sharingGroup, c4), false).get();
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 6), sharingGroup, c1), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 6), sharingGroup, c2), false).get();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 6), sharingGroup, c3), false).get();
        SimpleSlot s8 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 6), sharingGroup, c5), false).get();
        SimpleSlot s9 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 6), sharingGroup, c6), false).get();
        SimpleSlot s10 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 6), sharingGroup, c4), false).get();
        SimpleSlot s11 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 6), sharingGroup, c5), false).get();
        SimpleSlot s12 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 6), sharingGroup, c6), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertNotNull(s5);
        assertNotNull(s6);
        assertNotNull(s7);
        assertNotNull(s8);
        assertNotNull(s9);
        assertNotNull(s10);
        assertNotNull(s11);
        assertNotNull(s12);
        // check that each slot got exactly two tasks
        assertEquals(2, s1.getRoot().getNumberLeaves());
        assertEquals(2, s2.getRoot().getNumberLeaves());
        assertEquals(2, s3.getRoot().getNumberLeaves());
        assertEquals(2, s4.getRoot().getNumberLeaves());
        assertEquals(2, s5.getRoot().getNumberLeaves());
        assertEquals(2, s6.getRoot().getNumberLeaves());
        assertEquals(2, s7.getRoot().getNumberLeaves());
        assertEquals(2, s8.getRoot().getNumberLeaves());
        assertEquals(2, s9.getRoot().getNumberLeaves());
        assertEquals(2, s10.getRoot().getNumberLeaves());
        assertEquals(2, s11.getRoot().getNumberLeaves());
        assertEquals(2, s12.getRoot().getNumberLeaves());
        assertEquals(s1.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s6.getTaskManagerID());
        assertEquals(s3.getTaskManagerID(), s7.getTaskManagerID());
        assertEquals(s4.getTaskManagerID(), s10.getTaskManagerID());
        assertEquals(s8.getTaskManagerID(), s11.getTaskManagerID());
        assertEquals(s9.getTaskManagerID(), s12.getTaskManagerID());
        assertEquals(c1.getLocation(), s1.getTaskManagerLocation());
        assertEquals(c2.getLocation(), s2.getTaskManagerLocation());
        assertEquals(c3.getLocation(), s3.getTaskManagerLocation());
        assertEquals(c4.getLocation(), s4.getTaskManagerLocation());
        assertEquals(c5.getLocation(), s8.getTaskManagerLocation());
        assertEquals(c6.getLocation(), s9.getTaskManagerLocation());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        // the first assignments are unconstrained, co.-scheduling is constrained
        assertEquals(6, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(6, scheduler.getNumberOfUnconstrainedAssignments());
        // release some slots, be sure that new available ones come up
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        s7.releaseSlot();
        s10.releaseSlot();
        s11.releaseSlot();
        s12.releaseSlot();
        assertTrue(scheduler.getNumberOfAvailableSlots() >= 1);
        SimpleSlot single = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(new JobVertexID(), 0, 1)), false).get();
        assertNotNull(single);
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        s7.releaseSlot();
        s8.releaseSlot();
        s9.releaseSlot();
        s11.releaseSlot();
        s12.releaseSlot();
        assertEquals(5, scheduler.getNumberOfAvailableSlots());
        assertEquals(6, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(7, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 69 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleWithIntermediateRelease.

@Test
public void scheduleWithIntermediateRelease() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        JobVertexID jid4 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1), sharingGroup, c1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1), sharingGroup, c1), false).get();
        SimpleSlot sSolo = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1)), false).get();
        ResourceID taskManager = s1.getTaskManagerID();
        s1.releaseSlot();
        s2.releaseSlot();
        sSolo.releaseSlot();
        SimpleSlot sNew = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup, c1), false).get();
        assertEquals(taskManager, sNew.getTaskManagerID());
        assertEquals(2, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 70 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleWithReleaseNoResource.

@Test
public void scheduleWithReleaseNoResource() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1), sharingGroup, c1), false).get();
        s1.releaseSlot();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1)), false).get();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2)), false).get();
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup, c1), false).get();
            fail("Scheduled even though no resource was available.");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        }
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(3, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}