Examples with SimpleSlot org.apache.flink.runtime.instance.SimpleSlot used on opensource projects

Example 71 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerIsolatedTasksTest method testSchedulingLocation.

@Test
public void testSchedulingLocation() {
    try {
        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        Instance i3 = getRandomInstance(2);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i3);
        // schedule something on an arbitrary instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false).get();
        // figure out how we use the location hints
        Instance first = (Instance) s1.getOwner();
        Instance second = first != i1 ? i1 : i2;
        Instance third = first == i3 ? i2 : i3;
        // something that needs to go to the first instance again
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false).get();
        assertEquals(first, s2.getOwner());
        // first or second --> second, because first is full
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false).get();
        assertEquals(second, s3.getOwner());
        // first or third --> third (because first is full)
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(third, s4.getOwner());
        assertEquals(third, s5.getOwner());
        // first or third --> second, because all others are full
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(second, s6.getOwner());
        // release something on the first and second instance
        s2.releaseSlot();
        s6.releaseSlot();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false).get();
        assertEquals(first, s7.getOwner());
        assertEquals(1, scheduler.getNumberOfUnconstrainedAssignments());
        assertEquals(1, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(5, scheduler.getNumberOfLocalizedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 72 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerIsolatedTasksTest method testScheduleQueueing.

@Test
public void testScheduleQueueing() {
    final int NUM_INSTANCES = 50;
    final int NUM_SLOTS_PER_INSTANCE = 3;
    final int NUM_TASKS_TO_SCHEDULE = 2000;
    try {
        // note: since this test asynchronously releases slots, the executor needs release workers.
        // doing the release call synchronous can lead to a deadlock
        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        for (int i = 0; i < NUM_INSTANCES; i++) {
            scheduler.newInstanceAvailable(getRandomInstance((int) (Math.random() * NUM_SLOTS_PER_INSTANCE) + 1));
        }
        assertEquals(NUM_INSTANCES, scheduler.getNumberOfAvailableInstances());
        final int totalSlots = scheduler.getNumberOfAvailableSlots();
        // all slots we ever got.
        List<Future<SimpleSlot>> allAllocatedSlots = new ArrayList<>();
        // slots that need to be released
        final Set<SimpleSlot> toRelease = new HashSet<SimpleSlot>();
        // flag to track errors in the concurrent thread
        final AtomicBoolean errored = new AtomicBoolean(false);
        // thread to asynchronously release slots
        Runnable disposer = new Runnable() {

            @Override
            public void run() {
                try {
                    int recycled = 0;
                    while (recycled < NUM_TASKS_TO_SCHEDULE) {
                        synchronized (toRelease) {
                            while (toRelease.isEmpty()) {
                                toRelease.wait();
                            }
                            Iterator<SimpleSlot> iter = toRelease.iterator();
                            SimpleSlot next = iter.next();
                            iter.remove();
                            next.releaseSlot();
                            recycled++;
                        }
                    }
                } catch (Throwable t) {
                    errored.set(true);
                }
            }
        };
        Thread disposeThread = new Thread(disposer);
        disposeThread.start();
        for (int i = 0; i < NUM_TASKS_TO_SCHEDULE; i++) {
            Future<SimpleSlot> future = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), true);
            future.thenAcceptAsync(new AcceptFunction<SimpleSlot>() {

                @Override
                public void accept(SimpleSlot slot) {
                    synchronized (toRelease) {
                        toRelease.add(slot);
                        toRelease.notifyAll();
                    }
                }
            }, TestingUtils.defaultExecutionContext());
            allAllocatedSlots.add(future);
        }
        disposeThread.join();
        assertFalse("The slot releasing thread caused an error.", errored.get());
        List<SimpleSlot> slotsAfter = new ArrayList<SimpleSlot>();
        for (Future<SimpleSlot> future : allAllocatedSlots) {
            slotsAfter.add(future.get());
        }
        assertEquals("All instances should have available slots.", NUM_INSTANCES, scheduler.getNumberOfInstancesWithAvailableSlots());
        // the slots should all be different
        assertTrue(areAllDistinct(slotsAfter.toArray()));
        assertEquals("All slots should be available.", totalSlots, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 73 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment1.

/**
	 * Tests that the scheduler assigns the correct existing shared slots
	 */
@Test
public void testLocalizedAssignment1() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule one to each instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, i1.getNumberOfAvailableSlots());
        assertEquals(1, i2.getNumberOfAvailableSlots());
        // schedule one from the other group to each instance
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s3);
        assertNotNull(s4);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, i1.getNumberOfAvailableSlots());
        assertEquals(1, i2.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 74 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method scheduleMixedSharingAndNonSharing.

@Test
public void scheduleMixedSharingAndNonSharing() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jidA = new JobVertexID();
        JobVertexID jidB = new JobVertexID();
        JobVertexID jidC = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(3));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule some individual vertices
        SimpleSlot sA1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 0, 2)), false).get();
        SimpleSlot sA2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 1, 2)), false).get();
        assertNotNull(sA1);
        assertNotNull(sA2);
        // schedule some vertices in the sharing group
        SimpleSlot s1_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot s1_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot s2_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4), sharingGroup), false).get();
        assertNotNull(s1_0);
        assertNotNull(s1_1);
        assertNotNull(s2_0);
        assertNotNull(s2_1);
        // schedule another isolated vertex
        SimpleSlot sB1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 1, 3)), false).get();
        assertNotNull(sB1);
        // should not be able to schedule more vertices
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3)), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 1)), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release some isolated task and check that the sharing group may grow
        sA1.releaseSlot();
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        // release three of the previously allocated sub slots, which guarantees to return one shared slot
        s1_0.releaseSlot();
        s1_1.releaseSlot();
        s2_0.releaseSlot();
        assertEquals(1, scheduler.getNumberOfAvailableSlots());
        // schedule one more no-shared task
        SimpleSlot sB0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3)), false).get();
        assertNotNull(sB0);
        // release the last of the original shared slots and allocate one more non-shared slot
        s2_1.releaseSlot();
        SimpleSlot sB2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 2, 3)), false).get();
        assertNotNull(sB2);
        // release on non-shared and add some shared slots
        sA2.releaseSlot();
        SimpleSlot s1_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        SimpleSlot s2_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), sharingGroup), false).get();
        assertNotNull(s1_3);
        assertNotNull(s2_3);
        // release all shared and allocate all in non-shared
        s1_2.releaseSlot();
        s2_2.releaseSlot();
        s1_3.releaseSlot();
        s2_3.releaseSlot();
        SimpleSlot sC0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 1, 2)), false).get();
        SimpleSlot sC1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 2)), false).get();
        assertNotNull(sC0);
        assertNotNull(sC1);
        sB0.releaseSlot();
        sB1.releaseSlot();
        sB2.releaseSlot();
        sC0.releaseSlot();
        sC1.releaseSlot();
        // test that everything is released
        assertEquals(5, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(15, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 75 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithIntermediateTotallyEmptySharingGroup.

@Test
public void allocateSlotWithIntermediateTotallyEmptySharingGroup() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        s1.releaseSlot();
        s2.releaseSlot();
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // schedule some tasks from the second ID group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4), sharingGroup), false).get();
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        s1_2.releaseSlot();
        s2_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(8, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}