use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.
the class SharedSlotsTest method testReleaseTwoLevelsFromRoot.
/**
* We allocate and the structure below and release it from the root.
*
* <pre>
* Shared(0)(root)
* |
* +-- Simple(2)(sink)
* |
* +-- Shared(1)(co-location-group)
* | |
* | +-- Simple(0)(tail)
* | +-- Simple(1)(head)
* |
* +-- Simple(0)(source)
* </pre>
*/
@Test
public void testReleaseTwoLevelsFromRoot() {
try {
JobVertexID sourceId = new JobVertexID();
JobVertexID headId = new JobVertexID();
JobVertexID tailId = new JobVertexID();
JobVertexID sinkId = new JobVertexID();
JobVertex headVertex = new JobVertex("head", headId);
JobVertex tailVertex = new JobVertex("tail", tailId);
SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
assertEquals(0, assignment.getNumberOfSlots());
CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
assertFalse(constraint.isAssigned());
Instance instance = SchedulerTestUtils.getRandomInstance(1);
// allocate a shared slot
SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
// get the first simple slot
SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
SimpleSlot headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
constraint.lockLocation();
SimpleSlot tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, NO_LOCATION);
assertEquals(4, sharedSlot.getNumberLeaves());
// release all
sourceSlot.releaseSlot();
headSlot.releaseSlot();
tailSlot.releaseSlot();
sinkSlot.releaseSlot();
assertTrue(sharedSlot.isReleased());
assertTrue(sourceSlot.isReleased());
assertTrue(headSlot.isReleased());
assertTrue(tailSlot.isReleased());
assertTrue(sinkSlot.isReleased());
assertTrue(constraint.getSharedSlot().isReleased());
assertTrue(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
assertEquals(1, instance.getNumberOfAvailableSlots());
assertEquals(0, instance.getNumberOfAllocatedSlots());
assertEquals(0, assignment.getNumberOfSlots());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.
the class SharedSlotsTest method testAllocateAndReleaseTwoLevels.
/**
* We allocate and release the structure below, starting by allocating a simple slot in the
* shared slot and finishing by releasing a simple slot.
*
* <pre>
* Shared(0)(root)
* |
* +-- Simple(2)(sink)
* |
* +-- Shared(1)(co-location-group)
* | |
* | +-- Simple(0)(tail)
* | +-- Simple(1)(head)
* |
* +-- Simple(0)(source)
* </pre>
*/
@Test
public void testAllocateAndReleaseTwoLevels() {
try {
JobVertexID sourceId = new JobVertexID();
JobVertexID headId = new JobVertexID();
JobVertexID tailId = new JobVertexID();
JobVertexID sinkId = new JobVertexID();
JobVertex headVertex = new JobVertex("head", headId);
JobVertex tailVertex = new JobVertex("tail", tailId);
SlotSharingGroup sharingGroup = new SlotSharingGroup(sourceId, headId, tailId, sinkId);
SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
assertEquals(0, assignment.getNumberOfSlots());
CoLocationGroup coLocationGroup = new CoLocationGroup(headVertex, tailVertex);
CoLocationConstraint constraint = coLocationGroup.getLocationConstraint(0);
assertFalse(constraint.isAssigned());
Instance instance = SchedulerTestUtils.getRandomInstance(1);
// allocate a shared slot
SharedSlot sharedSlot = instance.allocateSharedSlot(new JobID(), assignment);
// get the first simple slot
SimpleSlot sourceSlot = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.LOCAL, sourceId);
assertEquals(1, sharedSlot.getNumberLeaves());
// get the first slot in the nested shared slot from the co-location constraint
SimpleSlot headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
assertEquals(2, sharedSlot.getNumberLeaves());
assertNotNull(constraint.getSharedSlot());
assertTrue(constraint.getSharedSlot().isAlive());
assertFalse(constraint.isAssigned());
// we do not immediately lock the location
headSlot.releaseSlot();
assertEquals(1, sharedSlot.getNumberLeaves());
assertNotNull(constraint.getSharedSlot());
assertTrue(constraint.getSharedSlot().isReleased());
assertFalse(constraint.isAssigned());
// re-allocate the head slot
headSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
constraint.lockLocation();
assertNotNull(constraint.getSharedSlot());
assertTrue(constraint.isAssigned());
assertTrue(constraint.isAssignedAndAlive());
assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
SimpleSlot tailSlot = assignment.getSlotForTask(constraint, Collections.<TaskManagerLocation>emptySet());
assertEquals(constraint.getSharedSlot(), headSlot.getParent());
assertEquals(constraint.getSharedSlot(), tailSlot.getParent());
SimpleSlot sinkSlot = assignment.getSlotForTask(sinkId, Collections.<TaskManagerLocation>emptySet());
assertEquals(4, sharedSlot.getNumberLeaves());
// we release our co-location constraint tasks
headSlot.releaseSlot();
tailSlot.releaseSlot();
assertEquals(2, sharedSlot.getNumberLeaves());
assertTrue(headSlot.isReleased());
assertTrue(tailSlot.isReleased());
assertTrue(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
assertEquals(instance.getTaskManagerLocation(), constraint.getLocation());
// we should have resources again for the co-location constraint
assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
// re-allocate head and tail from the constraint
headSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
tailSlot = assignment.getSlotForTask(constraint, NO_LOCATION);
assertEquals(4, sharedSlot.getNumberLeaves());
assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(constraint.getGroupId()));
// verify some basic properties of the slots
assertEquals(instance.getTaskManagerID(), sourceSlot.getTaskManagerID());
assertEquals(instance.getTaskManagerID(), headSlot.getTaskManagerID());
assertEquals(instance.getTaskManagerID(), tailSlot.getTaskManagerID());
assertEquals(instance.getTaskManagerID(), sinkSlot.getTaskManagerID());
assertEquals(sourceId, sourceSlot.getGroupID());
assertEquals(sinkId, sinkSlot.getGroupID());
assertNull(headSlot.getGroupID());
assertNull(tailSlot.getGroupID());
assertEquals(constraint.getGroupId(), constraint.getSharedSlot().getGroupID());
// release all
sourceSlot.releaseSlot();
headSlot.releaseSlot();
tailSlot.releaseSlot();
sinkSlot.releaseSlot();
assertTrue(sharedSlot.isReleased());
assertTrue(sourceSlot.isReleased());
assertTrue(headSlot.isReleased());
assertTrue(tailSlot.isReleased());
assertTrue(sinkSlot.isReleased());
assertTrue(constraint.getSharedSlot().isReleased());
assertTrue(constraint.isAssigned());
assertFalse(constraint.isAssignedAndAlive());
assertEquals(1, instance.getNumberOfAvailableSlots());
assertEquals(0, assignment.getNumberOfSlots());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.
the class LocalInputPreferredSlotSharingStrategyTest method testCoLocationConstraintIsRespected.
@Test
public void testCoLocationConstraintIsRespected() {
topology.connect(ev11, ev22);
topology.connect(ev12, ev21);
final CoLocationGroup coLocationGroup = new TestingCoLocationGroup(JOB_VERTEX_ID_1, JOB_VERTEX_ID_2);
final Set<CoLocationGroup> coLocationGroups = Collections.singleton(coLocationGroup);
final SlotSharingStrategy strategy = new LocalInputPreferredSlotSharingStrategy(topology, slotSharingGroups, coLocationGroups);
assertThat(strategy.getExecutionSlotSharingGroups(), hasSize(2));
assertThat(strategy.getExecutionSlotSharingGroup(ev11.getId()).getExecutionVertexIds(), containsInAnyOrder(ev11.getId(), ev21.getId()));
assertThat(strategy.getExecutionSlotSharingGroup(ev12.getId()).getExecutionVertexIds(), containsInAnyOrder(ev12.getId(), ev22.getId()));
}
use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.
the class StreamingJobGraphGeneratorTest method testIteration.
/**
* Test iteration job, check slot sharing group and co-location group.
*/
@Test
public void testIteration() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Integer> source = env.fromElements(1, 2, 3).name("source");
IterativeStream<Integer> iteration = source.iterate(3000);
iteration.name("iteration").setParallelism(2);
DataStream<Integer> map = iteration.map(x -> x + 1).name("map").setParallelism(2);
DataStream<Integer> filter = map.filter((x) -> false).name("filter").setParallelism(2);
iteration.closeWith(filter).print();
JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
SlotSharingGroup slotSharingGroup = jobGraph.getVerticesAsArray()[0].getSlotSharingGroup();
assertNotNull(slotSharingGroup);
CoLocationGroup iterationSourceCoLocationGroup = null;
CoLocationGroup iterationSinkCoLocationGroup = null;
for (JobVertex jobVertex : jobGraph.getVertices()) {
// all vertices have same slot sharing group by default
assertEquals(slotSharingGroup, jobVertex.getSlotSharingGroup());
// others have no co-location group by default
if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SOURCE_NAME_PREFIX)) {
iterationSourceCoLocationGroup = jobVertex.getCoLocationGroup();
assertTrue(iterationSourceCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
} else if (jobVertex.getName().startsWith(StreamGraph.ITERATION_SINK_NAME_PREFIX)) {
iterationSinkCoLocationGroup = jobVertex.getCoLocationGroup();
assertTrue(iterationSinkCoLocationGroup.getVertexIds().contains(jobVertex.getID()));
} else {
assertNull(jobVertex.getCoLocationGroup());
}
}
assertNotNull(iterationSourceCoLocationGroup);
assertNotNull(iterationSinkCoLocationGroup);
assertEquals(iterationSourceCoLocationGroup, iterationSinkCoLocationGroup);
}
use of org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup in project flink by apache.
the class JobGraphTest method testGetCoLocationGroups.
@Test
public void testGetCoLocationGroups() {
final JobVertex v1 = new JobVertex("1");
final JobVertex v2 = new JobVertex("2");
final JobVertex v3 = new JobVertex("3");
final JobVertex v4 = new JobVertex("4");
final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
v1.setSlotSharingGroup(slotSharingGroup);
v2.setSlotSharingGroup(slotSharingGroup);
v1.setStrictlyCoLocatedWith(v2);
final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(Arrays.asList(v1, v2, v3, v4)).build();
assertThat(jobGraph.getCoLocationGroups(), hasSize(1));
final CoLocationGroup onlyCoLocationGroup = jobGraph.getCoLocationGroups().iterator().next();
assertThat(onlyCoLocationGroup.getVertexIds(), containsInAnyOrder(v1.getID(), v2.getID()));
}
Aggregations