Examples with SlotSharingGroup org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup used on opensource projects

Example 26 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class JobExecutionITCase method createJobGraph.

private JobGraph createJobGraph(int parallelism) {
    final JobVertex sender = new JobVertex("Sender");
    sender.setParallelism(parallelism);
    sender.setInvokableClass(TestingAbstractInvokables.Sender.class);
    final JobVertex receiver = new JobVertex("Receiver");
    receiver.setParallelism(parallelism);
    receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);
    // In order to make testCoLocationConstraintJobExecution fail, one needs to
    // remove the co-location constraint and the slot sharing groups, because then
    // the receivers will have to wait for the senders to finish and the slot
    // assignment order to the receivers is non-deterministic (depending on the
    // order in which the senders finish).
    final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    receiver.setSlotSharingGroup(slotSharingGroup);
    sender.setSlotSharingGroup(slotSharingGroup);
    receiver.setStrictlyCoLocatedWith(sender);
    receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    return JobGraphTestUtils.streamingJobGraph(sender, receiver);
}

Example 27 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class DefaultExecutionTopologyTest method testErrorIfCoLocatedTasksAreNotInSameRegion.

@Test(expected = IllegalStateException.class)
public void testErrorIfCoLocatedTasksAreNotInSameRegion() throws Exception {
    int parallelism = 3;
    final JobVertex v1 = createNoOpVertex(parallelism);
    final JobVertex v2 = createNoOpVertex(parallelism);
    SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
    v1.setSlotSharingGroup(slotSharingGroup);
    v2.setSlotSharingGroup(slotSharingGroup);
    v1.setStrictlyCoLocatedWith(v2);
    final DefaultExecutionGraph executionGraph = createSimpleTestGraph(v1, v2);
    DefaultExecutionTopology.fromExecutionGraph(executionGraph);
}

Example 28 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SsgNetworkMemoryCalculationUtilsTest method testGenerateEnrichedResourceProfileForDynamicGraph.

@Test
public void testGenerateEnrichedResourceProfileForDynamicGraph() throws Exception {
    List<SlotSharingGroup> slotSharingGroups = Arrays.asList(new SlotSharingGroup(), new SlotSharingGroup(), new SlotSharingGroup());
    for (SlotSharingGroup group : slotSharingGroups) {
        group.setResourceProfile(DEFAULT_RESOURCE);
    }
    DefaultExecutionGraph executionGraph = createDynamicExecutionGraph(slotSharingGroups, 20);
    Iterator<ExecutionJobVertex> jobVertices = executionGraph.getVerticesTopologically().iterator();
    ExecutionJobVertex source = jobVertices.next();
    ExecutionJobVertex map = jobVertices.next();
    ExecutionJobVertex sink = jobVertices.next();
    executionGraph.initializeJobVertex(source, 0L);
    triggerComputeNumOfSubpartitions(source.getProducedDataSets()[0]);
    map.setParallelism(5);
    executionGraph.initializeJobVertex(map, 0L);
    triggerComputeNumOfSubpartitions(map.getProducedDataSets()[0]);
    sink.setParallelism(7);
    executionGraph.initializeJobVertex(sink, 0L);
    assertNetworkMemory(slotSharingGroups, Arrays.asList(new MemorySize(TestShuffleMaster.computeRequiredShuffleMemoryBytes(0, 5)), new MemorySize(TestShuffleMaster.computeRequiredShuffleMemoryBytes(5, 20)), new MemorySize(TestShuffleMaster.computeRequiredShuffleMemoryBytes(15, 0))));
}

Example 29 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SharedSlotsTest method allocateAndReleaseInMixedOrder.

@Test
public void allocateAndReleaseInMixedOrder() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid1);
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        assertNotNull(sub1);
        assertNotNull(sub2);
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        sub2.releaseSlot();
        assertEquals(1, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, NO_LOCATION);
        assertNotNull(sub3);
        assertEquals(2, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        sub3.releaseSlot();
        sub1.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertEquals(0, sharedSlot.getNumberLeaves());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 30 with SlotSharingGroup

use of org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup in project flink by apache.

the class SharedSlotsTest method allocateSimpleSlotsAndReleaseFromLeaves.

@Test
public void allocateSimpleSlotsAndReleaseFromLeaves() {
    try {
        JobID jobId = new JobID();
        JobVertexID vid1 = new JobVertexID();
        JobVertexID vid2 = new JobVertexID();
        JobVertexID vid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(vid1, vid2, vid3);
        SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
        Instance instance = SchedulerTestUtils.getRandomInstance(1);
        // allocate a shared slot
        SharedSlot sharedSlot = instance.allocateSharedSlot(jobId, assignment);
        // allocate a series of sub slots
        SimpleSlot sub1 = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid1);
        SimpleSlot sub2 = assignment.getSlotForTask(vid2, NO_LOCATION);
        SimpleSlot sub3 = assignment.getSlotForTask(vid3, NO_LOCATION);
        assertNotNull(sub1);
        assertNotNull(sub2);
        assertNotNull(sub3);
        assertEquals(3, sharedSlot.getNumberLeaves());
        assertEquals(1, assignment.getNumberOfSlots());
        // release from the leaves.
        sub2.releaseSlot();
        assertTrue(sharedSlot.isAlive());
        assertTrue(sub1.isAlive());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isAlive());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        assertEquals(2, sharedSlot.getNumberLeaves());
        sub1.releaseSlot();
        assertTrue(sharedSlot.isAlive());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isAlive());
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(1, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(1, assignment.getNumberOfSlots());
        assertEquals(1, sharedSlot.getNumberLeaves());
        sub3.releaseSlot();
        assertTrue(sharedSlot.isReleased());
        assertTrue(sub1.isReleased());
        assertTrue(sub2.isReleased());
        assertTrue(sub3.isReleased());
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid1));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid2));
        assertEquals(0, assignment.getNumberOfAvailableSlotsForGroup(vid3));
        assertEquals(0, assignment.getNumberOfSlots());
        assertEquals(1, instance.getNumberOfAvailableSlots());
        assertEquals(0, assignment.getNumberOfSlots());
        assertNull(sharedSlot.allocateSharedSlot(new AbstractID()));
        assertNull(sharedSlot.allocateSubSlot(new AbstractID()));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}