Examples with NettyShuffleEnvironmentBuilder org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder used on opensource projects

Example 11 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class TaskSubmissionTestEnvironment method createShuffleEnvironment.

private static ShuffleEnvironment<?, ?> createShuffleEnvironment(ResourceID taskManagerLocation, boolean localCommunication, Configuration configuration, RpcService testingRpcService, boolean mockShuffleEnvironment) throws Exception {
    final ShuffleEnvironment<?, ?> shuffleEnvironment;
    if (mockShuffleEnvironment) {
        shuffleEnvironment = mock(ShuffleEnvironment.class, Mockito.RETURNS_MOCKS);
    } else {
        final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getByName(testingRpcService.getAddress()), configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT));
        final NettyConfig nettyConfig = new NettyConfig(socketAddress.getAddress(), socketAddress.getPort(), ConfigurationParserUtils.getPageSize(configuration), ConfigurationParserUtils.getSlot(configuration), configuration);
        shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setTaskManagerLocation(taskManagerLocation).setPartitionRequestInitialBackoff(configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL)).setPartitionRequestMaxBackoff(configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX)).setNettyConfig(localCommunication ? null : nettyConfig).build();
        shuffleEnvironment.start();
    }
    return shuffleEnvironment;
}

Example 12 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class AbstractUdfStreamOperatorLifecycleTest method testLifeCycleCancel.

@Test
public void testLifeCycleCancel() throws Exception {
    ACTUAL_ORDER_TRACKING.clear();
    Configuration taskManagerConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(new Configuration());
    MockSourceFunction srcFun = new MockSourceFunction();
    cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
    cfg.setOperatorID(new OperatorID());
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (ShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = StreamTaskTest.createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        task.startTaskThread();
        LifecycleTrackingStreamSource.runStarted.await();
        // this should cancel the task even though it is blocked on runFinished
        task.cancelExecution();
        // wait for clean termination
        task.getExecutingThread().join();
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
        assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
    }
}

Example 13 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class UnalignedCheckpointsTest method createInputGate.

private CheckpointedInputGate createInputGate(int numberOfChannels, AbstractInvokable toNotify, boolean enableCheckpointsAfterTasksFinished) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setStateWriter(channelStateWriter).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    sequenceNumbers = new int[numberOfChannels];
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify, enableCheckpointsAfterTasksFinished);
}

Example 14 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class AlignedCheckpointsTest method createCheckpointedInputGate.

private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}

Example 15 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class StreamTaskTest method testTaskAvoidHangingAfterSnapshotStateThrownException.

@Test
public void testTaskAvoidHangingAfterSnapshotStateThrownException() throws Exception {
    // given: Configured SourceStreamTask with source which fails on checkpoint.
    Configuration taskManagerConfig = new Configuration();
    taskManagerConfig.setString(STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateKeySerializer(mock(TypeSerializer.class));
    cfg.setOperatorID(new OperatorID(4712L, 43L));
    FailedSource failedSource = new FailedSource();
    cfg.setStreamOperator(new TestStreamSource<String, FailedSource>(failedSource));
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        // when: Task starts
        task.startTaskThread();
        // wait for the task starts doing the work.
        failedSource.awaitRunning();
        // and: Checkpoint is triggered which should lead to exception in Source.
        task.triggerCheckpointBarrier(42L, 1L, CheckpointOptions.forCheckpointWithDefaultLocation());
        // wait for clean termination.
        task.getExecutingThread().join();
        // then: The task doesn't hang but finished with FAILED state.
        assertEquals(ExecutionState.FAILED, task.getExecutionState());
    }
}