Examples with NettyShuffleEnvironmentBuilder org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder used on opensource projects

Example 31 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class TaskTest method setup.

@Before
public void setup() {
    awaitLatch = new OneShotLatch();
    triggerLatch = new OneShotLatch();
    shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    wasCleanedUp = false;
}

Example 32 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class AbstractUdfStreamOperatorLifecycleTest method testLifeCycleFull.

@Test
public void testLifeCycleFull() throws Exception {
    ACTUAL_ORDER_TRACKING.clear();
    Configuration taskManagerConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(new Configuration());
    MockSourceFunction srcFun = new MockSourceFunction();
    cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
    cfg.setOperatorID(new OperatorID());
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (ShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = StreamTaskTest.createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        task.startTaskThread();
        LifecycleTrackingStreamSource.runStarted.await();
        // wait for clean termination
        task.getExecutingThread().join();
        assertEquals(ExecutionState.FINISHED, task.getExecutionState());
        assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
    }
}

Example 33 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class NettyShuffleUtilsTest method testComputeRequiredNetworkBuffers.

/**
 * This test verifies that the {@link NettyShuffleEnvironment} requires buffers as expected, so
 * that the required shuffle memory size returned by {@link
 * ShuffleMaster#computeShuffleMemorySizeForTask(TaskInputsOutputsDescriptor)} is correct.
 */
@Test
public void testComputeRequiredNetworkBuffers() throws Exception {
    int numBuffersPerChannel = 5;
    int numBuffersPerGate = 8;
    int sortShuffleMinParallelism = 8;
    int numSortShuffleMinBuffers = 12;
    int numChannels1 = 3;
    int numChannels2 = 4;
    IntermediateDataSetID ds1 = new IntermediateDataSetID();
    IntermediateDataSetID ds2 = new IntermediateDataSetID();
    IntermediateDataSetID ds3 = new IntermediateDataSetID();
    // pipelined shuffle
    int numSubs1 = 5;
    // hash blocking shuffle
    int numSubs2 = 6;
    // sort blocking shuffle
    int numSubs3 = 10;
    Map<IntermediateDataSetID, Integer> subpartitionNums = ImmutableMap.of(ds1, numSubs1, ds2, numSubs2, ds3, numSubs3);
    Map<IntermediateDataSetID, ResultPartitionType> partitionTypes = ImmutableMap.of(ds1, PIPELINED_BOUNDED, ds2, BLOCKING, ds3, BLOCKING);
    int numTotalBuffers = NettyShuffleUtils.computeNetworkBuffersForAnnouncing(numBuffersPerChannel, numBuffersPerGate, sortShuffleMinParallelism, numSortShuffleMinBuffers, numChannels1 + numChannels2, 2, subpartitionNums, partitionTypes);
    NettyShuffleEnvironment sEnv = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(numTotalBuffers).setNetworkBuffersPerChannel(numBuffersPerChannel).setSortShuffleMinBuffers(numSortShuffleMinBuffers).setSortShuffleMinParallelism(sortShuffleMinParallelism).build();
    SingleInputGate inputGate1 = createInputGate(sEnv, PIPELINED_BOUNDED, numChannels1);
    inputGate1.setup();
    SingleInputGate inputGate2 = createInputGate(sEnv, BLOCKING, numChannels2);
    inputGate2.setup();
    ResultPartition resultPartition1 = createResultPartition(sEnv, PIPELINED_BOUNDED, numSubs1);
    resultPartition1.setup();
    ResultPartition resultPartition2 = createResultPartition(sEnv, BLOCKING, numSubs2);
    resultPartition2.setup();
    ResultPartition resultPartition3 = createResultPartition(sEnv, BLOCKING, numSubs3);
    resultPartition3.setup();
    int expected = calculateBuffersConsumption(inputGate1) + calculateBuffersConsumption(inputGate2) + calculateBuffersConsumption(resultPartition1) + calculateBuffersConsumption(resultPartition2) + calculateBuffersConsumption(resultPartition3);
    assertEquals(expected, numTotalBuffers);
    inputGate1.close();
    inputGate2.close();
    resultPartition1.close();
    resultPartition2.close();
    resultPartition3.close();
}

Example 34 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class CheckpointBarrierTrackerTest method createCheckpointedInputGate.

// ------------------------------------------------------------------------
// Utils
// ------------------------------------------------------------------------
private CheckpointedInputGate createCheckpointedInputGate(int numberOfChannels, AbstractInvokable toNotify) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify);
}

Example 35 with NettyShuffleEnvironmentBuilder

use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method testPartitionRelease.

private void testPartitionRelease(PartitionTrackerSetup partitionTrackerSetup, TestAction testAction) throws Exception {
    final TestingTaskExecutorPartitionTracker partitionTracker = new TestingTaskExecutorPartitionTracker();
    final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
    partitionTracker.setStartTrackingPartitionsConsumer((jobId, partitionInfo) -> startTrackingFuture.complete(partitionInfo.getResultPartitionId()));
    partitionTrackerSetup.accept(partitionTracker);
    internalTestPartitionRelease(partitionTracker, new NettyShuffleEnvironmentBuilder().build(), startTrackingFuture, testAction);
}