Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 81 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testConcurrentAllocateAndRelease.

@Test
public void testConcurrentAllocateAndRelease() {
    final ExecutorService executor = Executors.newFixedThreadPool(20);
    try {
        for (int run = 0; run < 50; run++) {
            final JobVertexID jid1 = new JobVertexID();
            final JobVertexID jid2 = new JobVertexID();
            final JobVertexID jid3 = new JobVertexID();
            final JobVertexID jid4 = new JobVertexID();
            final SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3, jid4);
            final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
            scheduler.newInstanceAvailable(getRandomInstance(4));
            final AtomicInteger enumerator1 = new AtomicInteger();
            final AtomicInteger enumerator2 = new AtomicInteger();
            final AtomicBoolean flag3 = new AtomicBoolean();
            final AtomicInteger enumerator4 = new AtomicInteger();
            final Random rnd = new Random();
            // use atomic boolean as a mutable boolean reference
            final AtomicBoolean failed = new AtomicBoolean(false);
            // use atomic integer as a mutable integer reference
            final AtomicInteger completed = new AtomicInteger();
            final Runnable deploy4 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4), sharingGroup), false).get();
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy3 = new Runnable() {

                @Override
                public void run() {
                    try {
                        if (flag3.compareAndSet(false, true)) {
                            SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup), false).get();
                            sleepUninterruptibly(5);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            slot.releaseSlot();
                            if (completed.incrementAndGet() == 13) {
                                synchronized (completed) {
                                    completed.notifyAll();
                                }
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy2 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4), sharingGroup), false).get();
                        // wait a bit till scheduling the successor
                        sleepUninterruptibly(rnd.nextInt(5));
                        executor.execute(deploy3);
                        // wait a bit until release
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy1 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4), sharingGroup), false).get();
                        // wait a bit till scheduling the successor
                        sleepUninterruptibly(rnd.nextInt(5));
                        executor.execute(deploy2);
                        // wait a bit until release
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy0 = new Runnable() {

                @Override
                public void run() {
                    sleepUninterruptibly(rnd.nextInt(10));
                    executor.execute(deploy1);
                }
            };
            executor.execute(deploy0);
            executor.execute(deploy0);
            executor.execute(deploy0);
            executor.execute(deploy0);
            //noinspection SynchronizationOnLocalVariableOrMethodParameter
            synchronized (completed) {
                while (!failed.get() && completed.get() < 13) {
                    completed.wait(1000);
                }
            }
            assertFalse("Thread failed", failed.get());
            while (scheduler.getNumberOfAvailableSlots() < 4) {
                sleepUninterruptibly(5);
            }
            assertEquals(1, scheduler.getNumberOfAvailableInstances());
            assertEquals(1, scheduler.getNumberOfInstancesWithAvailableSlots());
            assertEquals(4, scheduler.getNumberOfAvailableSlots());
            assertEquals(13, scheduler.getNumberOfUnconstrainedAssignments());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 82 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithTemprarilyEmptyVertexGroup.

@Test
public void allocateSlotWithTemprarilyEmptyVertexGroup() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_1);
        assertNotNull(s2_1);
        assertNotNull(s3_1);
        assertNotNull(s4_1);
        assertTrue(areAllDistinct(s1_1, s2_1, s3_1, s4_1));
        // schedule 4 tasks from the second vertex group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        assertNotNull(s3_2);
        assertNotNull(s4_2);
        assertTrue(areAllDistinct(s1_2, s2_2, s3_2, s4_2));
        // schedule 4 tasks from the third vertex group
        SimpleSlot s1_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_3);
        assertNotNull(s2_3);
        assertNotNull(s3_3);
        assertNotNull(s4_3);
        assertTrue(areAllDistinct(s1_3, s2_3, s3_3, s4_3));
        // we cannot schedule another task from the second vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release the second vertex group
        s1_2.releaseSlot();
        s2_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        SimpleSlot s5_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7), sharingGroup), false).get();
        SimpleSlot s6_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7), sharingGroup), false).get();
        SimpleSlot s7_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7), sharingGroup), false).get();
        assertNotNull(s5_2);
        assertNotNull(s6_2);
        assertNotNull(s7_2);
        // release the slots
        s1_1.releaseSlot();
        s2_1.releaseSlot();
        s3_1.releaseSlot();
        s4_1.releaseSlot();
        s5_2.releaseSlot();
        s6_2.releaseSlot();
        s7_2.releaseSlot();
        // test that everything is released
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        s1_3.releaseSlot();
        s2_3.releaseSlot();
        s3_3.releaseSlot();
        s4_3.releaseSlot();
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(15, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 83 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithSharing.

@Test
public void allocateSlotWithSharing() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertTrue(areAllDistinct(s1, s2, s3, s4));
        // we cannot schedule another task from the first vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // schedule some tasks from the second ID group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        assertNotNull(s3_2);
        assertNotNull(s4_2);
        // we cannot schedule another task from the second vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // now, we release some vertices (sub-slots) from the first group.
        // that should allow us to schedule more vertices from the first group
        s1.releaseSlot();
        s4.releaseSlot();
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // we can still not schedule anything from the second group of vertices
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // we can schedule something from the first vertex group
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
        assertNotNull(s5);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // now we release a slot from the second vertex group and schedule another task from that group
        s2_2.releaseSlot();
        SimpleSlot s5_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
        assertNotNull(s5_2);
        // release all slots
        s2.releaseSlot();
        s3.releaseSlot();
        s5.releaseSlot();
        s1_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        s5_2.releaseSlot();
        // test that everything is released
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(10, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 84 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment3.

/**
	 * Tests that the scheduler can fall back to non-local
	 */
@Test
public void testLocalizedAssignment3() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule until the one instance is full
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, loc1), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, loc1), sharingGroup), false).get();
        // schedule two more with preference of same instance --> need to go to other instance
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, loc1), sharingGroup), false).get();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertNotNull(s5);
        assertNotNull(s6);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        assertEquals(i1.getTaskManagerID(), s1.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s2.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s3.getTaskManagerID());
        assertEquals(i1.getTaskManagerID(), s4.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s5.getTaskManagerID());
        assertEquals(i2.getTaskManagerID(), s6.getTaskManagerID());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(2, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 85 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class SchedulerSlotSharingTest method testLocalizedAssignment2.

/**
	 * Tests that the scheduler assigns to new local slots, rather than to existing non-local slots
	 */
@Test
public void testLocalizedAssignment2() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        TaskManagerLocation loc1 = i1.getTaskManagerLocation();
        TaskManagerLocation loc2 = i2.getTaskManagerLocation();
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule one to each instance
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, loc1), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, loc1), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(2, i2.getNumberOfAvailableSlots());
        // schedule one from the other group to each instance
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, loc2), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, loc2), sharingGroup), false).get();
        assertNotNull(s3);
        assertNotNull(s4);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(0, i1.getNumberOfAvailableSlots());
        assertEquals(0, i2.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(4, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}