Examples with ExecutionException java.util.concurrent.ExecutionException used on opensource projects

Example 6 with ExecutionException

use of java.util.concurrent.ExecutionException in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithTemprarilyEmptyVertexGroup.

@Test
public void allocateSlotWithTemprarilyEmptyVertexGroup() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        JobVertexID jid3 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2, jid3);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_1);
        assertNotNull(s2_1);
        assertNotNull(s3_1);
        assertNotNull(s4_1);
        assertTrue(areAllDistinct(s1_1, s2_1, s3_1, s4_1));
        // schedule 4 tasks from the second vertex group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        assertNotNull(s3_2);
        assertNotNull(s4_2);
        assertTrue(areAllDistinct(s1_2, s2_2, s3_2, s4_2));
        // schedule 4 tasks from the third vertex group
        SimpleSlot s1_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4), sharingGroup), false).get();
        SimpleSlot s2_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4), sharingGroup), false).get();
        SimpleSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4), sharingGroup), false).get();
        SimpleSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4), sharingGroup), false).get();
        assertNotNull(s1_3);
        assertNotNull(s2_3);
        assertNotNull(s3_3);
        assertNotNull(s4_3);
        assertTrue(areAllDistinct(s1_3, s2_3, s3_3, s4_3));
        // we cannot schedule another task from the second vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // release the second vertex group
        s1_2.releaseSlot();
        s2_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        SimpleSlot s5_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7), sharingGroup), false).get();
        SimpleSlot s6_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7), sharingGroup), false).get();
        SimpleSlot s7_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7), sharingGroup), false).get();
        assertNotNull(s5_2);
        assertNotNull(s6_2);
        assertNotNull(s7_2);
        // release the slots
        s1_1.releaseSlot();
        s2_1.releaseSlot();
        s3_1.releaseSlot();
        s4_1.releaseSlot();
        s5_2.releaseSlot();
        s6_2.releaseSlot();
        s7_2.releaseSlot();
        // test that everything is released
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        s1_3.releaseSlot();
        s2_3.releaseSlot();
        s3_3.releaseSlot();
        s4_3.releaseSlot();
        // test that everything is released
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(15, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with ExecutionException

use of java.util.concurrent.ExecutionException in project flink by apache.

the class SchedulerSlotSharingTest method allocateSlotWithSharing.

@Test
public void allocateSlotWithSharing() {
    try {
        JobVertexID jid1 = new JobVertexID();
        JobVertexID jid2 = new JobVertexID();
        SlotSharingGroup sharingGroup = new SlotSharingGroup(jid1, jid2);
        Scheduler scheduler = new Scheduler(TestingUtils.directExecutionContext());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        // schedule 4 tasks from the first vertex group
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5), sharingGroup), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5), sharingGroup), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5), sharingGroup), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5), sharingGroup), false).get();
        assertNotNull(s1);
        assertNotNull(s2);
        assertNotNull(s3);
        assertNotNull(s4);
        assertTrue(areAllDistinct(s1, s2, s3, s4));
        // we cannot schedule another task from the first vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // schedule some tasks from the second ID group
        SimpleSlot s1_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5), sharingGroup), false).get();
        SimpleSlot s2_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5), sharingGroup), false).get();
        SimpleSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5), sharingGroup), false).get();
        SimpleSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5), sharingGroup), false).get();
        assertNotNull(s1_2);
        assertNotNull(s2_2);
        assertNotNull(s3_2);
        assertNotNull(s4_2);
        // we cannot schedule another task from the second vertex group
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // now, we release some vertices (sub-slots) from the first group.
        // that should allow us to schedule more vertices from the first group
        s1.releaseSlot();
        s4.releaseSlot();
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(2, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // we can still not schedule anything from the second group of vertices
        try {
            scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
            fail("Scheduler accepted too many tasks at the same time");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception.");
        }
        // we can schedule something from the first vertex group
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5), sharingGroup), false).get();
        assertNotNull(s5);
        assertEquals(4, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(1, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1));
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2));
        // now we release a slot from the second vertex group and schedule another task from that group
        s2_2.releaseSlot();
        SimpleSlot s5_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5), sharingGroup), false).get();
        assertNotNull(s5_2);
        // release all slots
        s2.releaseSlot();
        s3.releaseSlot();
        s5.releaseSlot();
        s1_2.releaseSlot();
        s3_2.releaseSlot();
        s4_2.releaseSlot();
        s5_2.releaseSlot();
        // test that everything is released
        assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
        assertEquals(4, scheduler.getNumberOfAvailableSlots());
        // check the scheduler's bookkeeping
        assertEquals(0, scheduler.getNumberOfLocalizedAssignments());
        assertEquals(0, scheduler.getNumberOfNonLocalizedAssignments());
        assertEquals(10, scheduler.getNumberOfUnconstrainedAssignments());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 8 with ExecutionException

use of java.util.concurrent.ExecutionException in project flink by apache.

the class SchedulerIsolatedTasksTest method testScheduleWithDyingInstances.

@Test
public void testScheduleWithDyingInstances() {
    try {
        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        Instance i1 = getRandomInstance(2);
        Instance i2 = getRandomInstance(2);
        Instance i3 = getRandomInstance(1);
        scheduler.newInstanceAvailable(i1);
        scheduler.newInstanceAvailable(i2);
        scheduler.newInstanceAvailable(i3);
        List<SimpleSlot> slots = new ArrayList<SimpleSlot>();
        slots.add(scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get());
        slots.add(scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get());
        slots.add(scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get());
        slots.add(scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get());
        slots.add(scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get());
        i2.markDead();
        for (SimpleSlot slot : slots) {
            if (slot.getOwner() == i2) {
                assertTrue(slot.isCanceled());
            } else {
                assertFalse(slot.isCanceled());
            }
            slot.releaseSlot();
        }
        assertEquals(3, scheduler.getNumberOfAvailableSlots());
        i1.markDead();
        i3.markDead();
        // cannot get another slot, since all instances are dead
        try {
            scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
            fail("Scheduler served a slot from a dead instance");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        } catch (Exception e) {
            fail("Wrong exception type.");
        }
        // now the latest, the scheduler should have noticed (through the lazy mechanisms)
        // that all instances have vanished
        assertEquals(0, scheduler.getNumberOfInstancesWithAvailableSlots());
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with ExecutionException

use of java.util.concurrent.ExecutionException in project flink by apache.

the class SchedulerIsolatedTasksTest method testScheduleImmediately.

@Test
public void testScheduleImmediately() {
    try {
        Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        assertEquals(0, scheduler.getNumberOfAvailableSlots());
        scheduler.newInstanceAvailable(getRandomInstance(2));
        scheduler.newInstanceAvailable(getRandomInstance(1));
        scheduler.newInstanceAvailable(getRandomInstance(2));
        assertEquals(5, scheduler.getNumberOfAvailableSlots());
        // schedule something into all slots
        SimpleSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        SimpleSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        SimpleSlot s3 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        SimpleSlot s4 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        SimpleSlot s5 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        // the slots should all be different
        assertTrue(areAllDistinct(s1, s2, s3, s4, s5));
        try {
            scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
            fail("Scheduler accepted scheduling request without available resource.");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof NoResourceAvailableException);
        }
        // release some slots again
        s3.releaseSlot();
        s4.releaseSlot();
        assertEquals(2, scheduler.getNumberOfAvailableSlots());
        // now we can schedule some more slots
        SimpleSlot s6 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        SimpleSlot s7 = scheduler.allocateSlot(new ScheduledUnit(getDummyTask()), false).get();
        assertTrue(areAllDistinct(s1, s2, s3, s4, s5, s6, s7));
        // release all
        s1.releaseSlot();
        s2.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        s7.releaseSlot();
        assertEquals(5, scheduler.getNumberOfAvailableSlots());
        // check that slots that are released twice (accidentally) do not mess things up
        s1.releaseSlot();
        s2.releaseSlot();
        s5.releaseSlot();
        s6.releaseSlot();
        s7.releaseSlot();
        assertEquals(5, scheduler.getNumberOfAvailableSlots());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with ExecutionException

use of java.util.concurrent.ExecutionException in project flink by apache.

the class StreamTaskTest method testFailingAsyncCheckpointRunnable.

/**
	 * Tests that in case of a failing AsyncCheckpointRunnable all operator snapshot results are
	 * cancelled and all non partitioned state handles are discarded.
	 */
@Test
public void testFailingAsyncCheckpointRunnable() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    StreamOperator<?> streamOperator1 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator2 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator3 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    // mock the new state handles / futures
    OperatorSnapshotResult operatorSnapshotResult1 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult2 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult3 = mock(OperatorSnapshotResult.class);
    RunnableFuture<OperatorStateHandle> failingFuture = mock(RunnableFuture.class);
    when(failingFuture.get()).thenThrow(new ExecutionException(new Exception("Test exception")));
    when(operatorSnapshotResult3.getOperatorStateRawFuture()).thenReturn(failingFuture);
    when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult1);
    when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult2);
    when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult3);
    // mock the legacy state snapshot
    StreamStateHandle streamStateHandle1 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle2 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle3 = mock(StreamStateHandle.class);
    when(streamOperator1.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle1);
    when(streamOperator2.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle2);
    when(streamOperator3.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle3);
    StreamOperator<?>[] streamOperators = { streamOperator1, streamOperator2, streamOperator3 };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", new DirectExecutorService());
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
    verify(streamTask).handleAsyncException(anyString(), any(Throwable.class));
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
    verify(operatorSnapshotResult3).cancel();
    verify(streamStateHandle1).discardState();
    verify(streamStateHandle2).discardState();
    verify(streamStateHandle3).discardState();
}