Examples with Instance org.apache.flink.runtime.instance.Instance used on opensource projects

Example 56 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetweenAndNoNewLocal.

@Test
public void testSlotReleasedInBetweenAndNoNewLocal() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jidx = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        s1.releaseSlot();
        s2.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        SimpleSlot sa = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2)), false).get();
        SimpleSlot sb = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2)), false).get();
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
            fail("should not be able to find a resource");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("wrong exception");
        }
        sa.releaseSlot();
        sb.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(2, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class ScheduleWithCoLocationHintTest method testGetsNonLocalFromSharingGroupFirst.

@Test
public void testGetsNonLocalFromSharingGroupFirst() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        // schedule something into the shared group so that both instances are in the sharing group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup), false).get();
        // schedule one locally to instance 1
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup, cc1), false).get();
        // schedule with co location constraint (yet unassigned) and a preference for
        // instance 1, but it can only get instance 2
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
        // schedule something into the assigned co-location constraints and check that they override the
        // other preferences
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, loc2), sharingGroup, cc1), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, loc1), sharingGroup, cc2), false).get();
        // check that each slot got three
        assertEquals(3, s1.getRoot().getNumberLeaves());
        assertEquals(3, s2.getRoot().getNumberLeaves());
        assertEquals(s1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(s1.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
        // release some slots, be sure that new available ones come up
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 58 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleWithIntermediateRelease.

@Test
public void scheduleWithIntermediateRelease() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        JobVertexID jid4 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1), sharingGroup, c1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1), sharingGroup, c1), false).get();
        SimpleSlot sSolo = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1)), false).get();
        ResourceID taskManager = s1.getTaskManagerID();
        s1.releaseSlot();
        s2.releaseSlot();
        sSolo.releaseSlot();
        SimpleSlot sNew = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup, c1), false).get();
        assertEquals(taskManager, sNew.getTaskManagerID());
        assertEquals(2, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 59 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class ScheduleWithCoLocationHintTest method scheduleWithReleaseNoResource.

@Test
public void scheduleWithReleaseNoResource() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1), sharingGroup, c1), false).get();
        s1.releaseSlot();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1)), false).get();
        scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2)), false).get();
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup, c1), false).get();
            fail("Scheduled even though no resource was available.");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        }
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(3, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 60 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class SchedulerIsolatedTasksTest method testSchedulingLocation.

@Test
public void testSchedulingLocation() {
    try {
        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        Instance i3 = getRandomInstance(2);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i3);
        // schedule something on an arbitrary instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false).get();
        // figure out how we use the location hints
        Instance first = (Instance) s1.getOwner();
        Instance second = first != i1 ? i1 : i2;
        Instance third = first == i3 ? i2 : i3;
        // something that needs to go to the first instance again
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false).get();
        assertEquals(first, s2.getOwner());
        // first or second --> second, because first is full
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false).get();
        assertEquals(second, s3.getOwner());
        // first or third --> third (because first is full)
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(third, s4.getOwner());
        assertEquals(third, s5.getOwner());
        // first or third --> second, because all others are full
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(second, s6.getOwner());
        // release something on the first and second instance
        s2.releaseSlot();
        s6.releaseSlot();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(first, s7.getOwner());
        assertEquals(1, scheduler.getNumberOfUnconstrainedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}