Examples with Instance org.apache.flink.runtime.instance.Instance used on opensource projects

Example 1 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class SchedulerSlotSharingTest method scheduleSingleVertexType.

@Test
public void scheduleSingleVertexType() {
    try {
        JobVertexID jid1 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertTrue(areAllDistinct(s1, s2, s3, s4));
        // we cannot schedule another task from the first vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release something
        s3.releaseSlot();
        // allocate another slot from that group
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
        assertNotNull(s5);
        // release all old slots
        s1.releaseSlot();
        s2.releaseSlot();
        s4.releaseSlot();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8), sharingGroup), false).get();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8), sharingGroup), false).get();
        SimpleSlot s8 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8), sharingGroup), false).get();
        assertNotNull(s6);
        assertNotNull(s7);
        assertNotNull(s8);
        // make sure we have two slots on the first instance, and two on the second
        int c = 0;
        c += (s5.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s6.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s7.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s8.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        assertEquals(0, c);
        // release all
        s5.releaseSlot();
        s6.releaseSlot();
        s7.releaseSlot();
        s8.releaseSlot();
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(8, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment3.

/**
	 * Tests that the scheduler can fall back to non-local
	 */
@Test
public void testLocalizedAssignment3() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule until the one instance is full
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, loc1), sharingGroup), false).get();
        // schedule two more with preference of same instance --> need to go to other instance
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, loc1), sharingGroup), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertNotNull(s5);
        assertNotNull(s6);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        assertEquals(i1.getTaskManagerID(), s1.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s2.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment2.

/**
	 * Tests that the scheduler assigns to new local slots, rather than to existing non-local slots
	 */
@Test
public void testLocalizedAssignment2() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule one to each instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(2, i2.getNumberOfAvailableSlots());
        // schedule one from the other group to each instance
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s3);
        assertNotNull(s4);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class CoLocationConstraintTest method testAssignSlotAndLockLocation.

@Test
public void testAssignSlotAndLockLocation() {
    try {
        JobID jid = new JobID();
        JobVertex vertex = new JobVertex("vertex");
        vertex.setParallelism(1);
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vertex.getID());
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        CoLocationGroup constraintGroup = new CoLocationGroup(vertex);
        CoLocationConstraint constraint = constraintGroup.getLocationConstraint(0);
        // constraint is completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        Instance instance1 = SchedulerTestUtils.getRandomInstance(2);
        Instance instance2 = SchedulerTestUtils.getRandomInstance(2);
        SharedSlot slot1_1 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot1_2 = instance1.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_1 = instance2.allocateSharedSlot(jid, assignment);
        SharedSlot slot2_2 = instance2.allocateSharedSlot(jid, assignment);
        // constraint is still completely unassigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // set the slot, but do not lock the location yet
        constraint.setSharedSlot(slot1_1);
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // try to get the location
        try {
            constraint.getLocation();
            fail("should throw an IllegalStateException");
        } catch (IllegalStateException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalStateException");
        }
        // check that we can reassign the slot as long as the location is not locked
        constraint.setSharedSlot(slot2_1);
        // the previous slot should have been released now
        assertTrue(slot1_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        // we can do an identity re-assign
        constraint.setSharedSlot(slot2_1);
        assertFalse(slot2_1.isReleased());
        // still the location is not assigned
        assertFalse(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        constraint.lockLocation();
        // now, the location is assigned and we have a location
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // release the slot
        slot2_1.releaseSlot();
        // we should still have a location
        assertTrue(constraint.isAssigned());
        assertFalse(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
        // we can not assign a different location
        try {
            constraint.setSharedSlot(slot1_2);
            fail("should throw an IllegalArgumentException");
        } catch (IllegalArgumentException e) {
        // as expected
        } catch (Exception e) {
            fail("wrong exception, should be IllegalArgumentException");
        }
        // assign a new slot with the same location
        constraint.setSharedSlot(slot2_2);
        assertTrue(constraint.isAssigned());
        assertTrue(constraint.isAssignedAndAlive());
        assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with Instance

use of org.apache.flink.runtime.instance.Instance in project flink by apache.

the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetween.

@Test
public void testSlotReleasedInBetween() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(1);
        Instance i2 = getRandomInstance(1);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i1);
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        SlotSharingGroup sharingGroup = new SlotSharingGroup();
        CoLocationGroup ccg = new CoLocationGroup();
        CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
        CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
        s1.releaseSlot();
        s2.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
        // still preserves the previous instance mapping)
        assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s4.getTaskManagerID());
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}