Examples with SimpleSlot org.apache.flink.runtime.instance.SimpleSlot used on opensource projects

Example 31 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class InputChannelDeploymentDescriptorTest method testUnknownChannelWithoutLazyDeploymentThrows.

@Test
public void testUnknownChannelWithoutLazyDeploymentThrows() throws Exception {
    ResourceID consumerResourceId = ResourceID.generate();
    ExecutionVertex consumer = mock(ExecutionVertex.class);
    SimpleSlot consumerSlot = mockSlot(consumerResourceId);
    // Unknown partition
    // no assigned resource
    ExecutionVertex unknownProducer = mockExecutionVertex(ExecutionState.CREATED, null);
    IntermediateResultPartition unknownPartition = mockPartition(unknownProducer);
    ResultPartitionID unknownPartitionId = new ResultPartitionID(unknownPartition.getPartitionId(), unknownProducer.getCurrentExecutionAttempt().getAttemptId());
    ExecutionEdge unknownEdge = new ExecutionEdge(unknownPartition, consumer, 2);
    // This should work if lazy deployment is allowed
    boolean allowLazyDeployment = true;
    InputChannelDeploymentDescriptor[] desc = InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
    assertEquals(1, desc.length);
    assertEquals(unknownPartitionId, desc[0].getConsumedPartitionId());
    assertTrue(desc[0].getConsumedPartitionLocation().isUnknown());
    assertNull(desc[0].getConsumedPartitionLocation().getConnectionId());
    try {
        // Fail if lazy deployment is *not* allowed
        allowLazyDeployment = false;
        InputChannelDeploymentDescriptor.fromEdges(new ExecutionEdge[] { unknownEdge }, consumerSlot, allowLazyDeployment);
        fail("Did not throw expected ExecutionGraphException");
    } catch (ExecutionGraphException ignored) {
    }
}

Example 32 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class InputChannelDeploymentDescriptorTest method mockExecutionVertex.

private static ExecutionVertex mockExecutionVertex(ExecutionState state, ResourceID resourceId) {
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    Execution exec = mock(Execution.class);
    when(exec.getState()).thenReturn(state);
    when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());
    if (resourceId != null) {
        SimpleSlot slot = mockSlot(resourceId);
        when(exec.getAssignedResource()).thenReturn(slot);
        when(vertex.getCurrentAssignedResource()).thenReturn(slot);
    } else {
        // no resource
        when(exec.getAssignedResource()).thenReturn(null);
        when(vertex.getCurrentAssignedResource()).thenReturn(null);
    }
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    return vertex;
}

Example 33 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method scheduleSingleVertexType.

@Test
public void scheduleSingleVertexType() {
    try {
        JobVertexID jid1 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertTrue(areAllDistinct(s1, s2, s3, s4));
        // we cannot schedule another task from the first vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release something
        s3.releaseSlot();
        // allocate another slot from that group
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8), sharingGroup), false).get();
        assertNotNull(s5);
        // release all old slots
        s1.releaseSlot();
        s2.releaseSlot();
        s4.releaseSlot();
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8), sharingGroup), false).get();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8), sharingGroup), false).get();
        SimpleSlot s8 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8), sharingGroup), false).get();
        assertNotNull(s6);
        assertNotNull(s7);
        assertNotNull(s8);
        // make sure we have two slots on the first instance, and two on the second
        int c = 0;
        c += (s5.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s6.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s7.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        c += (s8.getTaskManagerID().equals(i1.getTaskManagerID())) ? 1 : -1;
        assertEquals(0, c);
        // release all
        s5.releaseSlot();
        s6.releaseSlot();
        s7.releaseSlot();
        s8.releaseSlot();
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(8, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 34 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method testConcurrentAllocateAndRelease.

@Test
public void testConcurrentAllocateAndRelease() {
    final ExecutorService executor = Executors.newFixedThreadPool(20);
    try {
        for (int run = 0; run < 50; run++) {
            final JobVertexID jid1 = new JobVertexID();
            final JobVertexID jid2 = new JobVertexID();
            final JobVertexID jid3 = new JobVertexID();
            final JobVertexID jid4 = new JobVertexID();
            final SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3, jid4);
            final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
            scheduler.newInstanceAvailable(getRandomInstance(4));
            final AtomicInteger enumerator1 = new AtomicInteger();
            final AtomicInteger enumerator2 = new AtomicInteger();
            final AtomicBoolean flag3 = new AtomicBoolean();
            final AtomicInteger enumerator4 = new AtomicInteger();
            final Random rnd = new Random();
            // use atomic boolean as a mutable boolean reference
            final AtomicBoolean failed = new AtomicBoolean(false);
            // use atomic integer as a mutable integer reference
            final AtomicInteger completed = new AtomicInteger();
            final Runnable deploy4 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4), sharingGroup), false).get();
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy3 = new Runnable() {

                @Override
                public void run() {
                    try {
                        if (flag3.compareAndSet(false, true)) {
                            SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1), sharingGroup), false).get();
                            sleepUninterruptibly(5);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            executor.execute(deploy4);
                            slot.releaseSlot();
                            if (completed.incrementAndGet() == 13) {
                                synchronized (completed) {
                                    completed.notifyAll();
                                }
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy2 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4), sharingGroup), false).get();
                        // wait a bit till scheduling the successor
                        sleepUninterruptibly(rnd.nextInt(5));
                        executor.execute(deploy3);
                        // wait a bit until release
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy1 = new Runnable() {

                @Override
                public void run() {
                    try {
                        SimpleSlot slot = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4), sharingGroup), false).get();
                        // wait a bit till scheduling the successor
                        sleepUninterruptibly(rnd.nextInt(5));
                        executor.execute(deploy2);
                        // wait a bit until release
                        sleepUninterruptibly(rnd.nextInt(5));
                        slot.releaseSlot();
                        if (completed.incrementAndGet() == 13) {
                            synchronized (completed) {
                                completed.notifyAll();
                            }
                        }
                    } catch (Throwable t) {
                        t.printStackTrace();
                        failed.set(true);
                    }
                }
            };
            final Runnable deploy0 = new Runnable() {

                @Override
                public void run() {
                    sleepUninterruptibly(rnd.nextInt(10));
                    executor.execute(deploy1);
                }
            };
            executor.execute(deploy0);
            executor.execute(deploy0);
            executor.execute(deploy0);
            executor.execute(deploy0);
            //noinspection SynchronizationOnLocalVariableOrMethodParameter
            synchronized (completed) {
                while (!failed.get() && completed.get() < 13) {
                    completed.wait(1000);
                }
            }
            assertFalse("Thread failed", failed.get());
            while (scheduler.getNumberOfAvailableSlots() < 4) {
                sleepUninterruptibly(5);
            }
            assertEquals(1, scheduler.getNumberOfAvailableInstances());
            assertEquals(1, scheduler.getNumberOfInstancesWithAvailableSlots());
            assertEquals(4, scheduler.getNumberOfAvailableSlots());
            assertEquals(13, scheduler.getNumberOfUnconstrainedAssignments());
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 35 with SimpleSlot

use of org.apache.flink.runtime.instance.SimpleSlot in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithTemprarilyEmptyVertexGroup.

@Test
public void allocateSlotWithTemprarilyEmptyVertexGroup() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_1);
        assertNotNull(s2_1);
        assertNotNull(s3_1);
        assertNotNull(s4_1);
        assertTrue(areAllDistinct(s1_1, s2_1, s3_1, s4_1));
        // schedule 4 tasks from the second vertex group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        assertNotNull(s3_2);
        assertNotNull(s4_2);
        assertTrue(areAllDistinct(s1_2, s2_2, s3_2, s4_2));
        // schedule 4 tasks from the third vertex group
        SimpleSlot s1_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_3);
        assertNotNull(s2_3);
        assertNotNull(s3_3);
        assertNotNull(s4_3);
        assertTrue(areAllDistinct(s1_3, s2_3, s3_3, s4_3));
        // we cannot schedule another task from the second vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release the second vertex group
        s1_2.releaseSlot();
        s2_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        SimpleSlot s5_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7), sharingGroup), false).get();
        SimpleSlot s6_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7), sharingGroup), false).get();
        SimpleSlot s7_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7), sharingGroup), false).get();
        assertNotNull(s5_2);
        assertNotNull(s6_2);
        assertNotNull(s7_2);
        // release the slots
        s1_1.releaseSlot();
        s2_1.releaseSlot();
        s3_1.releaseSlot();
        s4_1.releaseSlot();
        s5_2.releaseSlot();
        s6_2.releaseSlot();
        s7_2.releaseSlot();
        // test that everything is released
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        s1_3.releaseSlot();
        s2_3.releaseSlot();
        s3_3.releaseSlot();
        s4_3.releaseSlot();
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(15, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}