use of org.apache.flink.runtime.instance.Instance in project flink by apache.
the class SchedulerSlotSharingTest method scheduleSingleVertexType.
@Test
public void scheduleSingleVertexType() {
try {
JobVertexID jid1 = new JobVertexID();
SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1);
Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
Instance i1 = getRandomInstance(2);
Instance i2 = getRandomInstance(2);
scheduler.newInstanceAvailable(i1);
scheduler.newInstanceAvailable(i2);
// schedule 4 tasks from the first vertex group
SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8), sharingGroup), false).get();
SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8), sharingGroup), false).get();
SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8), sharingGroup), false).get();
SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8), sharingGroup), false).get();
assertNotNull(s1);
assertNotNull(s2);
assertNotNull(s3);
assertNotNull(s4);
assertTrue(areAllDistinct(s1, s2, s3, s4));
// we cannot schedule another task from the first vertex group
try {
scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
fail("Scheduler accepted too many tasks at the same time");
} catch (ExecutionException e) {
assertTrue(e.getCause() instanceof NoResourceAvailableException);
} catch (Exception e) {
fail("Wrong exception.");
}
// release something
s3.releaseSlot();
// allocate another slot from that group
SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
assertNotNull(s5);
// release all old slots
s1.releaseSlot();
s2.releaseSlot();
s4.releaseSlot();
SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8), sharingGroup), false).get();
SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8), sharingGroup), false).get();
SimpleSlot s8 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8), sharingGroup), false).get();
assertNotNull(s6);
assertNotNull(s7);
assertNotNull(s8);
// make sure we have two slots on the first instance, and two on the second
int c = 0;
c += (s5.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
c += (s6.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
c += (s7.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
c += (s8.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
assertEquals(0, c);
// release all
s5.releaseSlot();
s6.releaseSlot();
s7.releaseSlot();
s8.releaseSlot();
// test that everything is released
assertEquals(4, scheduler.getNumberOfAvailableSlots());
// check the scheduler's bookkeeping
assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
assertEquals(8, scheduler.getNumberOfUnconstrainedAssignments());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.instance.Instance in project flink by apache.
the class SchedulerSlotSharingTest method testLocalizedAssignment3.
/**
* Tests that the scheduler can fall back to non-local
*/
@Test
public void testLocalizedAssignment3() {
try {
JobVertexID jid1 = new JobVertexID();
JobVertexID jid2 = new JobVertexID();
SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
Instance i1 = getRandomInstance(2);
Instance i2 = getRandomInstance(2);
TaskManagerLocation loc1 = i1.getTaskManagerLocation();
Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
scheduler.newInstanceAvailable(i1);
scheduler.newInstanceAvailable(i2);
// schedule until the one instance is full
SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, loc1), sharingGroup), false).get();
SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, loc1), sharingGroup), false).get();
// schedule two more with preference of same instance --> need to go to other instance
SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, loc1), sharingGroup), false).get();
SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, loc1), sharingGroup), false).get();
assertNotNull(s1);
assertNotNull(s2);
assertNotNull(s3);
assertNotNull(s4);
assertNotNull(s5);
assertNotNull(s6);
assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
assertEquals(0, i1.getNumberOfAvailableSlots());
assertEquals(0, i2.getNumberOfAvailableSlots());
assertEquals(i1.getTaskManagerID(), s1.getTaskManagerID());
assertEquals(i1.getTaskManagerID(), s2.getTaskManagerID());
assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
assertEquals(i1.getTaskManagerID(), s4.getTaskManagerID());
assertEquals(i2.getTaskManagerID(), s5.getTaskManagerID());
assertEquals(i2.getTaskManagerID(), s6.getTaskManagerID());
// check the scheduler's bookkeeping
assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
assertEquals(2, scheduler.getNumberOfNonLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.instance.Instance in project flink by apache.
the class SchedulerSlotSharingTest method testLocalizedAssignment2.
/**
* Tests that the scheduler assigns to new local slots, rather than to existing non-local slots
*/
@Test
public void testLocalizedAssignment2() {
try {
JobVertexID jid1 = new JobVertexID();
JobVertexID jid2 = new JobVertexID();
SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
Instance i1 = getRandomInstance(2);
Instance i2 = getRandomInstance(2);
TaskManagerLocation loc1 = i1.getTaskManagerLocation();
TaskManagerLocation loc2 = i2.getTaskManagerLocation();
Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
scheduler.newInstanceAvailable(i1);
scheduler.newInstanceAvailable(i2);
// schedule one to each instance
SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
assertNotNull(s1);
assertNotNull(s2);
assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
assertEquals(0, i1.getNumberOfAvailableSlots());
assertEquals(2, i2.getNumberOfAvailableSlots());
// schedule one from the other group to each instance
SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup), false).get();
SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
assertNotNull(s3);
assertNotNull(s4);
assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
assertEquals(0, i1.getNumberOfAvailableSlots());
assertEquals(0, i2.getNumberOfAvailableSlots());
// check the scheduler's bookkeeping
assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.instance.Instance in project flink by apache.
the class CoLocationConstraintTest method testAssignSlotAndLockLocation.
@Test
public void testAssignSlotAndLockLocation() {
try {
JobID jid = new JobID();
JobVertex vertex = new JobVertex("vertex");
vertex.setParallelism(1);
SlotSharingGroup sharingGroup = new SlotSharingGroup(vertex.getID());
SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
CoLocationGroup constraintGroup = new CoLocationGroup(vertex);
CoLocationConstraint constraint = constraintGroup.getLocationConstraint(0);
// constraint is completely unassigned
assertFalse(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
Instance instance1 = SchedulerTestUtils.getRandomInstance(2);
Instance instance2 = SchedulerTestUtils.getRandomInstance(2);
SharedSlot slot1_1 = instance1.allocateSharedSlot(jid, assignment);
SharedSlot slot1_2 = instance1.allocateSharedSlot(jid, assignment);
SharedSlot slot2_1 = instance2.allocateSharedSlot(jid, assignment);
SharedSlot slot2_2 = instance2.allocateSharedSlot(jid, assignment);
// constraint is still completely unassigned
assertFalse(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
// set the slot, but do not lock the location yet
constraint.setSharedSlot(slot1_1);
assertFalse(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
// try to get the location
try {
constraint.getLocation();
fail("should throw an IllegalStateException");
} catch (IllegalStateException e) {
// as expected
} catch (Exception e) {
fail("wrong exception, should be IllegalStateException");
}
// check that we can reassign the slot as long as the location is not locked
constraint.setSharedSlot(slot2_1);
// the previous slot should have been released now
assertTrue(slot1_1.isReleased());
// still the location is not assigned
assertFalse(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
// we can do an identity re-assign
constraint.setSharedSlot(slot2_1);
assertFalse(slot2_1.isReleased());
// still the location is not assigned
assertFalse(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
constraint.lockLocation();
// now, the location is assigned and we have a location
assertTrue(constraint.isAssigned());
assertTrue(constraint.isAssignedAndAlive());
assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
// release the slot
slot2_1.releaseSlot();
// we should still have a location
assertTrue(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
// we can not assign a different location
try {
constraint.setSharedSlot(slot1_2);
fail("should throw an IllegalArgumentException");
} catch (IllegalArgumentException e) {
// as expected
} catch (Exception e) {
fail("wrong exception, should be IllegalArgumentException");
}
// assign a new slot with the same location
constraint.setSharedSlot(slot2_2);
assertTrue(constraint.isAssigned());
assertTrue(constraint.isAssignedAndAlive());
assertEquals(instance2.getTaskManagerLocation(), constraint.getLocation());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.instance.Instance in project flink by apache.
the class ScheduleWithCoLocationHintTest method testSlotReleasedInBetween.
@Test
public void testSlotReleasedInBetween() {
try {
JobVertexID jid1 = new JobVertexID();
JobVertexID jid2 = new JobVertexID();
Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
Instance i1 = getRandomInstance(1);
Instance i2 = getRandomInstance(1);
TaskManagerLocation loc1 = i1.getTaskManagerLocation();
TaskManagerLocation loc2 = i2.getTaskManagerLocation();
scheduler.newInstanceAvailable(i2);
scheduler.newInstanceAvailable(i1);
assertEquals(2, scheduler.getNumberOfAvailableSlots());
SlotSharingGroup sharingGroup = new SlotSharingGroup();
CoLocationGroup ccg = new CoLocationGroup();
CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup, cc1), false).get();
SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup, cc2), false).get();
s1.releaseSlot();
s2.releaseSlot();
assertEquals(2, scheduler.getNumberOfAvailableSlots());
assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup, cc1), false).get();
SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc1), sharingGroup, cc2), false).get();
// still preserves the previous instance mapping)
assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
assertEquals(i2.getTaskManagerID(), s4.getTaskManagerID());
s3.releaseSlot();
s4.releaseSlot();
assertEquals(2, scheduler.getNumberOfAvailableSlots());
assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Aggregations