Examples with TaskManagerLocation org.apache.flink.runtime.taskmanager.TaskManagerLocation used on opensource projects

Example 71 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ScheduleWithCoLocationHintTest method testGetsNonLocalFromSharingGroupFirst.

@Test
public void testGetsNonLocalFromSharingGroupFirst() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        // schedule something into the shared group so that both instances are in the sharing group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup), false).get();
        // schedule one locally to instance 1
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup, cc1), false).get();
        // schedule with co location constraint (yet unassigned) and a preference for
        // instance 1, but it can only get instance 2
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
        // schedule something into the assigned co-location constraints and check that they override the
        // other preferences
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, loc2), sharingGroup, cc1), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, loc1), sharingGroup, cc2), false).get();
        // check that each slot got three
        assertEquals(3, s1.getRoot().getNumberLeaves());
        assertEquals(3, s2.getRoot().getNumberLeaves());
        assertEquals(s1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(s1.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
        // release some slots, be sure that new available ones come up
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 72 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment1.

/**
	 * Tests that the scheduler assigns the correct existing shared slots
	 */
@Test
public void testLocalizedAssignment1() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule one to each instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, i1.getNumberOfAvailableSlots());
        assertEquals(1, i2.getNumberOfAvailableSlots());
        // schedule one from the other group to each instance
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s3);
        assertNotNull(s4);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, i1.getNumberOfAvailableSlots());
        assertEquals(1, i2.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 73 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ScheduleWithCoLocationHintTest method nonColocationFollowsCoLocation.

@Test
public void nonColocationFollowsCoLocation() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup), false).get();
        // check that each slot got two
        assertEquals(2, s1.getRoot().getNumberLeaves());
        assertEquals(2, s2.getRoot().getNumberLeaves());
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 74 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ScheduleWithCoLocationHintTest method testScheduleOutOfOrder.

@Test
public void testScheduleOutOfOrder() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        // schedule something from the second job vertex id before the first is filled,
        // and give locality preferences that hint at using the same shared slot for both
        // co location constraints (which we seek to prevent)
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup, cc2), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup, cc2), false).get();
        // check that each slot got three
        assertEquals(2, s1.getRoot().getNumberLeaves());
        assertEquals(2, s2.getRoot().getNumberLeaves());
        assertEquals(s1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(s2.getTaskManagerID(), s4.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        assertEquals(3, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
        // release some slots, be sure that new available ones come up
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 75 with TaskManagerLocation

use of org.apache.flink.runtime.taskmanager.TaskManagerLocation in project flink by apache.

the class ExecutionGraphCoLocationRestartTest method validateConstraints.

private void validateConstraints(ExecutionGraph eg) {
    ExecutionJobVertex[] tasks = eg.getAllVertices().values().toArray(new ExecutionJobVertex[2]);
    for (int i = 0; i < NUM_TASKS; i++) {
        TaskManagerLocation taskManagerLocation0 = tasks[0].getTaskVertices()[i].getCurrentAssignedResourceLocation();
        TaskManagerLocation taskManagerLocation1 = tasks[1].getTaskVertices()[i].getCurrentAssignedResourceLocation();
        assertThat(taskManagerLocation0, is(taskManagerLocation1));
    }
}