Examples with DirectExecutorService org.apache.flink.runtime.util.DirectExecutorService used on opensource projects

Example 1 with DirectExecutorService

use of org.apache.flink.runtime.util.DirectExecutorService in project flink by apache.

the class StreamTaskTest method testFailingAsyncCheckpointRunnable.

/**
	 * Tests that in case of a failing AsyncCheckpointRunnable all operator snapshot results are
	 * cancelled and all non partitioned state handles are discarded.
	 */
@Test
public void testFailingAsyncCheckpointRunnable() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    StreamOperator<?> streamOperator1 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator2 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator3 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    // mock the new state handles / futures
    OperatorSnapshotResult operatorSnapshotResult1 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult2 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult3 = mock(OperatorSnapshotResult.class);
    RunnableFuture<OperatorStateHandle> failingFuture = mock(RunnableFuture.class);
    when(failingFuture.get()).thenThrow(new ExecutionException(new Exception("Test exception")));
    when(operatorSnapshotResult3.getOperatorStateRawFuture()).thenReturn(failingFuture);
    when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult1);
    when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult2);
    when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult3);
    // mock the legacy state snapshot
    StreamStateHandle streamStateHandle1 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle2 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle3 = mock(StreamStateHandle.class);
    when(streamOperator1.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle1);
    when(streamOperator2.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle2);
    when(streamOperator3.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle3);
    StreamOperator<?>[] streamOperators = { streamOperator1, streamOperator2, streamOperator3 };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", new DirectExecutorService());
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
    verify(streamTask).handleAsyncException(anyString(), any(Throwable.class));
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
    verify(operatorSnapshotResult3).cancel();
    verify(streamStateHandle1).discardState();
    verify(streamStateHandle2).discardState();
    verify(streamStateHandle3).discardState();
}

Example 2 with DirectExecutorService

use of org.apache.flink.runtime.util.DirectExecutorService in project flink by apache.

the class HeartbeatManagerTest method testHeartbeatCluster.

/**
	 * Tests the heartbeat interplay between the {@link HeartbeatManagerImpl} and the
	 * {@link HeartbeatManagerSenderImpl}. The sender should regularly trigger heartbeat requests
	 * which are fulfilled by the receiver. Upon stopping the receiver, the sender should notify
	 * the heartbeat listener about the heartbeat timeout.
	 *
	 * @throws Exception
	 */
@Test
public void testHeartbeatCluster() throws Exception {
    long heartbeatTimeout = 100L;
    long heartbeatPeriod = 20L;
    Object object = new Object();
    Object object2 = new Object();
    ResourceID resourceID = new ResourceID("foobar");
    ResourceID resourceID2 = new ResourceID("barfoo");
    HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
    when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(object));
    TestingHeartbeatListener heartbeatListener2 = new TestingHeartbeatListener(object2);
    Future<ResourceID> futureTimeout = heartbeatListener2.getTimeoutFuture();
    HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(heartbeatTimeout, resourceID, heartbeatListener, new DirectExecutorService(), new ScheduledExecutorServiceAdapter(new ScheduledThreadPoolExecutor(1)), LOG);
    HeartbeatManagerSenderImpl<Object, Object> heartbeatManager2 = new HeartbeatManagerSenderImpl<>(heartbeatPeriod, heartbeatTimeout, resourceID2, heartbeatListener2, new DirectExecutorService(), new ScheduledExecutorServiceAdapter(new ScheduledThreadPoolExecutor(1)), LOG);
    ;
    heartbeatManager.monitorTarget(resourceID2, heartbeatManager2);
    heartbeatManager2.monitorTarget(resourceID, heartbeatManager);
    Thread.sleep(2 * heartbeatTimeout);
    assertFalse(futureTimeout.isDone());
    heartbeatManager.stop();
    ResourceID timeoutResourceID = futureTimeout.get(2 * heartbeatTimeout, TimeUnit.MILLISECONDS);
    assertEquals(resourceID, timeoutResourceID);
    int numberHeartbeats = (int) (2 * heartbeatTimeout / heartbeatPeriod);
    verify(heartbeatListener, atLeast(numberHeartbeats / 2)).reportPayload(resourceID2, object2);
    assertTrue(heartbeatListener2.getNumberHeartbeatReports() >= numberHeartbeats / 2);
}

Example 3 with DirectExecutorService

use of org.apache.flink.runtime.util.DirectExecutorService in project flink by apache.

the class HeartbeatManagerTest method testHeartbeatMonitorUpdate.

/**
	 * Tests that the heartbeat monitors are updated when receiving a new heartbeat signal.
	 */
@Test
public void testHeartbeatMonitorUpdate() {
    long heartbeatTimeout = 1000L;
    ResourceID ownResourceID = new ResourceID("foobar");
    ResourceID targetResourceID = new ResourceID("barfoo");
    HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
    doReturn(scheduledFuture).when(scheduledExecutor).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
    Object expectedObject = new Object();
    when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(expectedObject));
    HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(heartbeatTimeout, ownResourceID, heartbeatListener, new DirectExecutorService(), scheduledExecutor, LOG);
    HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
    heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);
    heartbeatManager.receiveHeartbeat(targetResourceID, expectedObject);
    verify(scheduledFuture, times(1)).cancel(true);
    verify(scheduledExecutor, times(2)).schedule(any(Runnable.class), eq(heartbeatTimeout), eq(TimeUnit.MILLISECONDS));
}

Example 4 with DirectExecutorService

use of org.apache.flink.runtime.util.DirectExecutorService in project flink by apache.

the class HeartbeatManagerTest method testRegularHeartbeat.

/**
	 * Tests that regular heartbeat signal triggers the right callback functions in the
	 * {@link HeartbeatListener}.
	 */
@Test
public void testRegularHeartbeat() {
    long heartbeatTimeout = 1000L;
    ResourceID ownResourceID = new ResourceID("foobar");
    ResourceID targetResourceID = new ResourceID("barfoo");
    HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
    ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
    Object expectedObject = new Object();
    when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(expectedObject));
    HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(heartbeatTimeout, ownResourceID, heartbeatListener, new DirectExecutorService(), scheduledExecutor, LOG);
    HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
    heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);
    heartbeatManager.requestHeartbeat(targetResourceID, expectedObject);
    verify(heartbeatListener, times(1)).reportPayload(targetResourceID, expectedObject);
    verify(heartbeatListener, times(1)).retrievePayload();
    verify(heartbeatTarget, times(1)).receiveHeartbeat(ownResourceID, expectedObject);
    heartbeatManager.receiveHeartbeat(targetResourceID, expectedObject);
    verify(heartbeatListener, times(2)).reportPayload(targetResourceID, expectedObject);
}

Example 5 with DirectExecutorService

use of org.apache.flink.runtime.util.DirectExecutorService in project flink by apache.

the class HeartbeatManagerTest method testTargetUnmonitoring.

/**
	 * Tests that after unmonitoring a target, there won't be a timeout triggered
	 */
@Test
public void testTargetUnmonitoring() throws InterruptedException, ExecutionException {
    // this might be too aggresive for Travis, let's see...
    long heartbeatTimeout = 100L;
    ResourceID resourceID = new ResourceID("foobar");
    ResourceID targetID = new ResourceID("target");
    Object object = new Object();
    TestingHeartbeatListener heartbeatListener = new TestingHeartbeatListener(object);
    HeartbeatManager<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(heartbeatTimeout, resourceID, heartbeatListener, new DirectExecutorService(), new ScheduledExecutorServiceAdapter(new ScheduledThreadPoolExecutor(1)), LOG);
    heartbeatManager.monitorTarget(targetID, mock(HeartbeatTarget.class));
    heartbeatManager.unmonitorTarget(targetID);
    Future<ResourceID> timeout = heartbeatListener.getTimeoutFuture();
    try {
        timeout.get(2 * heartbeatTimeout, TimeUnit.MILLISECONDS);
        fail("Timeout should time out.");
    } catch (TimeoutException e) {
    // the timeout should not be completed since we unmonitored the target
    }
}