Examples with PartitionProducerStateChecker org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker used on opensource projects

Example 1 with PartitionProducerStateChecker

use of org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker in project flink by apache.

the class StreamTaskTest method createTask.

public static Task createTask(Class<? extends AbstractInvokable> invokable, StreamConfig taskConfig, Configuration taskManagerConfig) throws Exception {
    LibraryCacheManager libCache = mock(LibraryCacheManager.class);
    when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
    PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
    Executor executor = mock(Executor.class);
    NetworkEnvironment network = mock(NetworkEnvironment.class);
    when(network.getResultPartitionManager()).thenReturn(partitionManager);
    when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
    when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
    JobInformation jobInformation = new JobInformation(new JobID(), "Job Name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokable.getName(), taskConfig.getConfiguration());
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), 0, new TaskStateHandles(), mock(MemoryManager.class), mock(IOManager.class), network, mock(BroadcastVariableManager.class), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), libCache, mock(FileCache.class), new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] { System.getProperty("java.io.tmpdir") }), new UnregisteredTaskMetricsGroup(), consumableNotifier, partitionProducerStateChecker, executor);
}

Example 2 with PartitionProducerStateChecker

use of org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker in project flink by apache.

the class TaskAsyncCallTest method createTask.

private static Task createTask() throws Exception {
    LibraryCacheManager libCache = mock(LibraryCacheManager.class);
    when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
    PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
    Executor executor = mock(Executor.class);
    NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    when(networkEnvironment.getResultPartitionManager()).thenReturn(partitionManager);
    when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
    when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
    JobInformation jobInformation = new JobInformation(new JobID(), "Job Name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, CheckpointsInOrderInvokable.class.getName(), new Configuration());
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), 0, new TaskStateHandles(), mock(MemoryManager.class), mock(IOManager.class), networkEnvironment, mock(BroadcastVariableManager.class), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), libCache, mock(FileCache.class), new TestingTaskManagerRuntimeInfo(), mock(TaskMetricGroup.class), consumableNotifier, partitionProducerStateChecker, executor);
}

Example 3 with PartitionProducerStateChecker

use of org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker in project flink by apache.

the class TaskTest method testTriggerPartitionStateUpdate.

/**
	 * Tests the trigger partition state update future completions.
	 */
@Test
public void testTriggerPartitionStateUpdate() throws Exception {
    IntermediateDataSetID resultId = new IntermediateDataSetID();
    ResultPartitionID partitionId = new ResultPartitionID();
    LibraryCacheManager libCache = mock(LibraryCacheManager.class);
    when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader());
    PartitionProducerStateChecker partitionChecker = mock(PartitionProducerStateChecker.class);
    ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
    NetworkEnvironment network = mock(NetworkEnvironment.class);
    when(network.getResultPartitionManager()).thenReturn(mock(ResultPartitionManager.class));
    when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
    when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
    createTask(InvokableBlockingInInvoke.class, libCache, network, consumableNotifier, partitionChecker, Executors.directExecutor());
    // Test all branches of trigger partition state check
    {
        // Reset latches
        createQueuesAndActors();
        // PartitionProducerDisposedException
        Task task = createTask(InvokableBlockingInInvoke.class, libCache, network, consumableNotifier, partitionChecker, Executors.directExecutor());
        FlinkCompletableFuture<ExecutionState> promise = new FlinkCompletableFuture<>();
        when(partitionChecker.requestPartitionProducerState(eq(task.getJobID()), eq(resultId), eq(partitionId))).thenReturn(promise);
        task.triggerPartitionProducerStateCheck(task.getJobID(), resultId, partitionId);
        promise.completeExceptionally(new PartitionProducerDisposedException(partitionId));
        assertEquals(ExecutionState.CANCELING, task.getExecutionState());
    }
    {
        // Reset latches
        createQueuesAndActors();
        // Any other exception
        Task task = createTask(InvokableBlockingInInvoke.class, libCache, network, consumableNotifier, partitionChecker, Executors.directExecutor());
        FlinkCompletableFuture<ExecutionState> promise = new FlinkCompletableFuture<>();
        when(partitionChecker.requestPartitionProducerState(eq(task.getJobID()), eq(resultId), eq(partitionId))).thenReturn(promise);
        task.triggerPartitionProducerStateCheck(task.getJobID(), resultId, partitionId);
        promise.completeExceptionally(new RuntimeException("Any other exception"));
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
    }
    {
        // Reset latches
        createQueuesAndActors();
        // TimeoutException handled special => retry
        Task task = createTask(InvokableBlockingInInvoke.class, libCache, network, consumableNotifier, partitionChecker, Executors.directExecutor());
        SingleInputGate inputGate = mock(SingleInputGate.class);
        when(inputGate.getConsumedResultId()).thenReturn(resultId);
        try {
            task.startTaskThread();
            awaitLatch.await();
            setInputGate(task, inputGate);
            FlinkCompletableFuture<ExecutionState> promise = new FlinkCompletableFuture<>();
            when(partitionChecker.requestPartitionProducerState(eq(task.getJobID()), eq(resultId), eq(partitionId))).thenReturn(promise);
            task.triggerPartitionProducerStateCheck(task.getJobID(), resultId, partitionId);
            promise.completeExceptionally(new TimeoutException());
            assertEquals(ExecutionState.RUNNING, task.getExecutionState());
            verify(inputGate, times(1)).retriggerPartitionRequest(eq(partitionId.getPartitionId()));
        } finally {
            task.getExecutingThread().interrupt();
            task.getExecutingThread().join();
        }
    }
    {
        // Reset latches
        createQueuesAndActors();
        // Success
        Task task = createTask(InvokableBlockingInInvoke.class, libCache, network, consumableNotifier, partitionChecker, Executors.directExecutor());
        SingleInputGate inputGate = mock(SingleInputGate.class);
        when(inputGate.getConsumedResultId()).thenReturn(resultId);
        try {
            task.startTaskThread();
            awaitLatch.await();
            setInputGate(task, inputGate);
            FlinkCompletableFuture<ExecutionState> promise = new FlinkCompletableFuture<>();
            when(partitionChecker.requestPartitionProducerState(eq(task.getJobID()), eq(resultId), eq(partitionId))).thenReturn(promise);
            task.triggerPartitionProducerStateCheck(task.getJobID(), resultId, partitionId);
            promise.complete(ExecutionState.RUNNING);
            assertEquals(ExecutionState.RUNNING, task.getExecutionState());
            verify(inputGate, times(1)).retriggerPartitionRequest(eq(partitionId.getPartitionId()));
        } finally {
            task.getExecutingThread().interrupt();
            task.getExecutingThread().join();
        }
    }
}

Example 4 with PartitionProducerStateChecker

use of org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker in project flink by apache.

the class TaskTest method createTask.

private Task createTask(Class<? extends AbstractInvokable> invokable, LibraryCacheManager libCache, Configuration config, ExecutionConfig execConfig) throws IOException {
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
    PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
    Executor executor = mock(Executor.class);
    NetworkEnvironment network = mock(NetworkEnvironment.class);
    when(network.getResultPartitionManager()).thenReturn(partitionManager);
    when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
    when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
    return createTask(invokable, libCache, network, consumableNotifier, partitionProducerStateChecker, executor, config, execConfig);
}

Example 5 with PartitionProducerStateChecker

use of org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker in project flink by apache.

the class TaskTest method testExecutionFailsInNetworkRegistration.

@Test
public void testExecutionFailsInNetworkRegistration() {
    try {
        // mock a working library cache
        LibraryCacheManager libCache = mock(LibraryCacheManager.class);
        when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader());
        // mock a network manager that rejects registration
        ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
        ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
        PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
        Executor executor = mock(Executor.class);
        NetworkEnvironment network = mock(NetworkEnvironment.class);
        when(network.getResultPartitionManager()).thenReturn(partitionManager);
        when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
        doThrow(new RuntimeException("buffers")).when(network).registerTask(any(Task.class));
        Task task = createTask(TestInvokableCorrect.class, libCache, network, consumableNotifier, partitionProducerStateChecker, executor);
        task.registerExecutionListener(listener);
        task.run();
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
        assertTrue(task.isCanceledOrFailed());
        assertTrue(task.getFailureCause().getMessage().contains("buffers"));
        validateUnregisterTask(task.getExecutionId());
        validateListenerMessage(ExecutionState.FAILED, task, true);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}