Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 31 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyPartitionRequestClientTest method testAcknowledgeAllRecordsProcessed.

@Test
public void testAcknowledgeAllRecordsProcessed() throws Exception {
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        inputChannel.acknowledgeAllRecordsProcessed();
        channel.runPendingTasks();
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(NettyMessage.AckAllUserRecordsProcessed.class));
        assertEquals(inputChannel.getInputChannelId(), ((NettyMessage.AckAllUserRecordsProcessed) readFromOutbound).receiverId);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 32 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyPartitionRequestClientTest method testDoublePartitionRequest.

@Test
public void testDoublePartitionRequest() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // The input channel should only send one partition request
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 33 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyPartitionRequestClientTest method testRetriggerPartitionRequest.

@Test
public void testRetriggerPartitionRequest() throws Exception {
    // 30 secs
    final long deadline = System.currentTimeMillis() + 30_000L;
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().setConnectionManager(mockConnectionManagerWithPartitionRequestClient(client)).setInitialBackoff(1).setMaxBackoff(2).buildRemoteChannel(inputGate);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        // first subpartition request
        inputChannel.requestSubpartition();
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request once again, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 34 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class TaskManagerComponentsStartupShutdownTest method testComponentsStartupShutdown.

/**
	 * Makes sure that all components are shut down when the TaskManager
	 * actor is shut down.
	 */
@Test
public void testComponentsStartupShutdown() {
    final String[] TMP_DIR = new String[] { ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH };
    final Time timeout = Time.seconds(100);
    final int BUFFER_SIZE = 32 * 1024;
    Configuration config = new Configuration();
    config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "200 ms");
    config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "1 s");
    config.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 1);
    ActorSystem actorSystem = null;
    try {
        actorSystem = AkkaUtils.createLocalActorSystem(config);
        final ActorRef jobManager = JobManager.startJobManagerActors(config, actorSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), JobManager.class, MemoryArchivist.class)._1();
        FlinkResourceManager.startResourceManagerActors(config, actorSystem, LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager), StandaloneResourceManager.class);
        final int numberOfSlots = 1;
        // create the components for the TaskManager manually
        final TaskManagerConfiguration tmConfig = new TaskManagerConfiguration(numberOfSlots, TMP_DIR, timeout, null, Time.milliseconds(500), Time.seconds(30), Time.seconds(10), // cleanup interval
        1000000, config, // exit-jvm-on-fatal-error
        false);
        final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(32, BUFFER_SIZE, MemoryType.HEAP, IOManager.IOMode.SYNC, 0, 0, 2, 8, null);
        ResourceID taskManagerId = ResourceID.generate();
        final TaskManagerLocation connectionInfo = new TaskManagerLocation(taskManagerId, InetAddress.getLocalHost(), 10000);
        final MemoryManager memManager = new MemoryManager(32 * BUFFER_SIZE, 1, BUFFER_SIZE, MemoryType.HEAP, false);
        final IOManager ioManager = new IOManagerAsync(TMP_DIR);
        final NetworkEnvironment network = new NetworkEnvironment(new NetworkBufferPool(netConf.numNetworkBuffers(), netConf.networkBufferSize(), netConf.memoryType()), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, netConf.ioMode(), netConf.partitionRequestInitialBackoff(), netConf.partitionRequestMaxBackoff(), netConf.networkBuffersPerChannel(), netConf.extraNetworkBuffersPerGate());
        network.start();
        LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManager.path().toString());
        MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(config);
        // create the task manager
        final Props tmProps = Props.create(TaskManager.class, tmConfig, taskManagerId, connectionInfo, memManager, ioManager, network, numberOfSlots, leaderRetrievalService, new MetricRegistry(metricRegistryConfiguration));
        final ActorRef taskManager = actorSystem.actorOf(tmProps);
        new JavaTestKit(actorSystem) {

            {
                // wait for the TaskManager to be registered
                new Within(new FiniteDuration(5000, TimeUnit.SECONDS)) {

                    @Override
                    protected void run() {
                        taskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), getTestActor());
                        expectMsgEquals(TaskManagerMessages.getRegisteredAtJobManagerMessage());
                    }
                };
            }
        };
        // shut down all actors and the actor system
        // Kill the Task down the JobManager
        taskManager.tell(Kill.getInstance(), ActorRef.noSender());
        jobManager.tell(Kill.getInstance(), ActorRef.noSender());
        // shut down the actors and the actor system
        actorSystem.shutdown();
        actorSystem.awaitTermination();
        actorSystem = null;
        // now that the TaskManager is shut down, the components should be shut down as well
        assertTrue(network.isShutdown());
        assertTrue(ioManager.isProperlyShutDown());
        assertTrue(memManager.isShutdown());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (actorSystem != null) {
            actorSystem.shutdown();
        }
    }
}

Example 35 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class InputChannelRecoveredStateHandlerTest method setUp.

@Before
public void setUp() {
    // given: Segment provider with defined number of allocated segments.
    networkBufferPool = new NetworkBufferPool(preAllocatedSegments, 1024);
    // and: Configured input gate with recovered channel.
    inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildLocalRecoveredChannel).setSegmentProvider(networkBufferPool).build();
    icsHander = buildInputChannelStateHandler(inputGate);
    channelInfo = new InputChannelInfo(0, 0);
}