Examples with NettyShuffleEnvironment org.apache.flink.runtime.io.network.NettyShuffleEnvironment used on opensource projects

Example 11 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class UnalignedCheckpointsTest method createInputGate.

private CheckpointedInputGate createInputGate(int numberOfChannels, AbstractInvokable toNotify, boolean enableCheckpointsAfterTasksFinished) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setStateWriter(channelStateWriter).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    sequenceNumbers = new int[numberOfChannels];
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify, enableCheckpointsAfterTasksFinished);
}

Example 12 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class AlignedCheckpointsTest method createCheckpointedInputGate.

private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}

Example 13 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class StreamTaskTest method testTaskAvoidHangingAfterSnapshotStateThrownException.

@Test
public void testTaskAvoidHangingAfterSnapshotStateThrownException() throws Exception {
    // given: Configured SourceStreamTask with source which fails on checkpoint.
    Configuration taskManagerConfig = new Configuration();
    taskManagerConfig.setString(STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateKeySerializer(mock(TypeSerializer.class));
    cfg.setOperatorID(new OperatorID(4712L, 43L));
    FailedSource failedSource = new FailedSource();
    cfg.setStreamOperator(new TestStreamSource<String, FailedSource>(failedSource));
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        // when: Task starts
        task.startTaskThread();
        // wait for the task starts doing the work.
        failedSource.awaitRunning();
        // and: Checkpoint is triggered which should lead to exception in Source.
        task.triggerCheckpointBarrier(42L, 1L, CheckpointOptions.forCheckpointWithDefaultLocation());
        // wait for clean termination.
        task.getExecutingThread().join();
        // then: The task doesn't hang but finished with FAILED state.
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
    }
}

Example 14 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class StreamTaskTest method testRecordWriterClosedOnError.

private void testRecordWriterClosedOnError(FunctionWithException<NettyShuffleEnvironment, Task, Exception> taskProvider) throws Exception {
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = taskProvider.apply(shuffleEnvironment);
        task.startTaskThread();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
        for (Thread thread : Thread.getAllStackTraces().keySet()) {
            assertThat(thread.getName(), CoreMatchers.is(not(containsString(DEFAULT_OUTPUT_FLUSH_THREAD_NAME))));
        }
    }
}

Example 15 with NettyShuffleEnvironment

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironment in project flink by apache.

the class StreamTaskCancellationTest method testCancelTaskExceptionHandling.

/**
 * CancelTaskException can be thrown in a down stream task, for example if an upstream task was
 * cancelled first and those two tasks were connected via {@link
 * org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel}. {@link StreamTask}
 * should be able to correctly handle such situation.
 */
@Test
public void testCancelTaskExceptionHandling() throws Exception {
    StreamConfig cfg = new StreamConfig(new Configuration());
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(CancelThrowingTask.class, shuffleEnvironment, cfg, new Configuration());
        task.startTaskThread();
        task.getExecutingThread().join();
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    }
}