Examples with NettyShuffleEnvironmentBuilder org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder used on opensource projects

Example 16 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskTest method testRecordWriterClosedOnError.

private void testRecordWriterClosedOnError(FunctionWithException<NettyShuffleEnvironment, Task, Exception> taskProvider) throws Exception {
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = taskProvider.apply(shuffleEnvironment);
        task.startTaskThread();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            assertThat(thread.getName(), CoreMatchers.is(not(containsString(DEFAULT_OUTPUT_FLUSH_THREAD_NAME))));
        }
    }
}

Example 17 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskTerminationTest method testConcurrentAsyncCheckpointCannotFailFinishedStreamTask.

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing
 * operation after the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();
    final StateBackend blockingStateBackend = new BlockingStateBackend();
    streamConfig.setStreamOperator(noOpStreamOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStateBackend(blockingStateBackend);
    final long checkpointId = 0L;
    final long checkpointTimestamp = 0L;
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, BlockingStreamTask.class.getName(), taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
    CompletableFuture<Void> taskRun = CompletableFuture.runAsync(() -> task.run(), TestingUtils.defaultExecutor());
    // wait until the stream task started running
    RUN_LATCH.await();
    // trigger a checkpoint
    task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());
    // wait until the task has completed execution
    taskRun.get();
    // check that no failure occurred
    if (task.getFailureCause() != null) {
        throw new Exception("Task failed", task.getFailureCause());
    }
    // check that we have entered the finished state
    assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}

Example 18 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskCancellationTest method testCancelTaskExceptionHandling.

/**
 * CancelTaskException can be thrown in a down stream task, for example if an upstream task was
 * cancelled first and those two tasks were connected via {@link
 * org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel}. {@link StreamTask}
 * should be able to correctly handle such situation.
 */
@Test
public void testCancelTaskExceptionHandling() throws Exception {
    StreamConfig cfg = new StreamConfig(new Configuration());
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(CancelThrowingTask.class, shuffleEnvironment, cfg, new Configuration());
        task.startTaskThread();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    }
}

Example 19 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskCancellationTest method testCanceleablesCanceledOnCancelTaskError.

@Test
public void testCanceleablesCanceledOnCancelTaskError() throws Exception {
    CancelFailingTask.syncLatch = new OneShotLatch();
    StreamConfig cfg = new StreamConfig(new Configuration());
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(CancelFailingTask.class, shuffleEnvironment, cfg, new Configuration());
        // start the task and wait until it runs
        // execution state RUNNING is not enough, we need to wait until the stream task's run()
        // method
        // is entered
        task.startTaskThread();
        CancelFailingTask.syncLatch.await();
        // cancel the execution - this should lead to smooth shutdown
        task.cancelExecution();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    }
}

Example 20 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskTest method testStateBackendClosingOnFailure.

@Test
public void testStateBackendClosingOnFailure() throws Exception {
    Configuration taskManagerConfig = new Configuration();
    taskManagerConfig.setString(STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateKeySerializer(mock(TypeSerializer.class));
    cfg.setOperatorID(new OperatorID(4711L, 42L));
    TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
    cfg.setStreamOperator(streamSource);
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(StateBackendTestSource.class, shuffleEnvironment, cfg, taskManagerConfig);
        StateBackendTestSource.fail = true;
        task.startTaskThread();
        // wait for clean termination
        task.getExecutingThread().join();
        // ensure that the state backends and stream iterables are closed ...
        verify(TestStreamSource.operatorStateBackend).close();
        verify(TestStreamSource.keyedStateBackend).close();
        verify(TestStreamSource.rawOperatorStateInputs).close();
        verify(TestStreamSource.rawKeyedStateInputs).close();
        // ... and disposed
        verify(TestStreamSource.operatorStateBackend).dispose();
        verify(TestStreamSource.keyedStateBackend).dispose();
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
    }
}