Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 31 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyPartitionRequestClientTest method testAcknowledgeAllRecordsProcessed.

@Test
public void testAcknowledgeAllRecordsProcessed() throws Exception {
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        inputChannel.acknowledgeAllRecordsProcessed();
        channel.runPendingTasks();
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(NettyMessage.AckAllUserRecordsProcessed.class));
        assertEquals(inputChannel.getInputChannelId(), ((NettyMessage.AckAllUserRecordsProcessed) readFromOutbound).receiverId);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 32 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyPartitionRequestClientTest method testDoublePartitionRequest.

@Test
public void testDoublePartitionRequest() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // The input channel should only send one partition request
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 33 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyPartitionRequestClientTest method testRetriggerPartitionRequest.

@Test
public void testRetriggerPartitionRequest() throws Exception {
    // 30 secs
    final long deadline = System.currentTimeMillis() + 30_000L;
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().setConnectionManager(mockConnectionManagerWithPartitionRequestClient(client)).setInitialBackoff(1).setMaxBackoff(2).buildRemoteChannel(inputGate);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        // first subpartition request
        inputChannel.requestSubpartition();
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        // retrigger subpartition request once again, e.g. due to failures
        inputGate.retriggerPartitionRequest(inputChannel.getPartitionId().getPartitionId(), inputChannel.getConsumedSubpartitionIndex());
        runAllScheduledPendingTasks(channel, deadline);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannel.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(numExclusiveBuffers, ((PartitionRequest) readFromOutbound).credit);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 34 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NetworkEnvironment method registerTask.

// --------------------------------------------------------------------------------------------
//  Task operations
// --------------------------------------------------------------------------------------------
public void registerTask(Task task) throws IOException {
    final ResultPartition[] producedPartitions = task.getProducedPartitions();
    final ResultPartitionWriter[] writers = task.getAllWriters();
    if (writers.length != producedPartitions.length) {
        throw new IllegalStateException("Unequal number of writers and partitions.");
    }
    synchronized (lock) {
        if (isShutdown) {
            throw new IllegalStateException("NetworkEnvironment is shut down");
        }
        for (int i = 0; i < producedPartitions.length; i++) {
            final ResultPartition partition = producedPartitions[i];
            final ResultPartitionWriter writer = writers[i];
            // Buffer pool for the partition
            BufferPool bufferPool = null;
            try {
                int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE;
                bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments);
                partition.registerBufferPool(bufferPool);
                resultPartitionManager.registerResultPartition(partition);
            } catch (Throwable t) {
                if (bufferPool != null) {
                    bufferPool.lazyDestroy();
                }
                if (t instanceof IOException) {
                    throw (IOException) t;
                } else {
                    throw new IOException(t.getMessage(), t);
                }
            }
            // Register writer with task event dispatcher
            taskEventDispatcher.registerWriterForIncomingTaskEvents(writer.getPartitionId(), writer);
        }
        // Setup the buffer pool for each buffer reader
        final SingleInputGate[] inputGates = task.getAllInputGates();
        for (SingleInputGate gate : inputGates) {
            BufferPool bufferPool = null;
            try {
                int maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? gate.getNumberOfInputChannels() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE;
                bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), maxNumberOfMemorySegments);
                gate.setBufferPool(bufferPool);
            } catch (Throwable t) {
                if (bufferPool != null) {
                    bufferPool.lazyDestroy();
                }
                if (t instanceof IOException) {
                    throw (IOException) t;
                } else {
                    throw new IOException(t.getMessage(), t);
                }
            }
        }
    }
}

Example 35 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class TaskTest method setInputGate.

// ------------------------------------------------------------------------
private void setInputGate(Task task, SingleInputGate inputGate) {
    try {
        Field f = Task.class.getDeclaredField("inputGates");
        f.setAccessible(true);
        f.set(task, new SingleInputGate[] { inputGate });
        Map<IntermediateDataSetID, SingleInputGate> byId = new HashMap<>(1);
        byId.put(inputGate.getConsumedResultId(), inputGate);
        f = Task.class.getDeclaredField("inputGatesById");
        f.setAccessible(true);
        f.set(task, byId);
    } catch (Exception e) {
        throw new RuntimeException("Modifying the task state failed", e);
    }
}