Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 6 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestClientHandlerTest method testReceiveEmptyBuffer.

/**
	 * Tests a fix for FLINK-1761.
	 *
	 * <p> FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
	 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
    // Minimal mock of a remote input channel
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer());
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    // An empty buffer of size 0
    final Buffer emptyBuffer = TestBufferFactory.createBuffer();
    emptyBuffer.setSize(0);
    final BufferResponse receivedBuffer = createBufferResponse(emptyBuffer, 0, inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    // Read the empty buffer
    client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
    // This should not throw an exception
    verify(inputChannel, never()).onError(any(Throwable.class));
}

Example 7 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class InputChannelTestUtils method createMockBuffer.

/**
	 * Creates a simple Buffer that is not recycled (never will be) of the given size.
	 */
public static Buffer createMockBuffer(int size) {
    final Buffer mockBuffer = mock(Buffer.class);
    when(mockBuffer.isBuffer()).thenReturn(true);
    when(mockBuffer.getSize()).thenReturn(size);
    when(mockBuffer.isRecycled()).thenReturn(false);
    return mockBuffer;
}

Example 8 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannels.

@Test
public void testFairConsumptionRemoteChannels() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    final int[] channelSequenceNums = new int[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    channels[11].onBuffer(mockBuffer, 0);
    channelSequenceNums[11]++;
    // read all the buffers and the EOF event
    for (int i = 0; i < numChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
        if (i % (2 * numChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(channels, channelSequenceNums, 3, mockBuffer);
        }
    }
}

Example 9 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final PipelinedSubpartition[] sources = new PipelinedSubpartition[numChannels];
    for (int i = 0; i < numChannels; i++) {
        PipelinedSubpartition partition = new PipelinedSubpartition(0, resultPartition);
        for (int p = 0; p < buffersPerChannel; p++) {
            partition.add(mockBuffer);
        }
        partition.finish();
        sources[i] = partition;
    }
    // ----- create reading side -----
    ResultPartitionManager resultPartitionManager = createResultPartitionManager(sources);
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // read all the buffers and the EOF event
    for (int i = numChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getCurrentNumberOfBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
    }
    assertNull(gate.getNextBufferOrEvent());
}

Example 10 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannelsPreFilled.

@Test
public void testFairConsumptionRemoteChannelsPreFilled() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        channels[i] = channel;
        for (int p = 0; p < buffersPerChannel; p++) {
            channel.onBuffer(mockBuffer, p);
        }
        channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), buffersPerChannel);
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // read all the buffers and the EOF event
    for (int i = numChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
    }
    assertNull(gate.getNextBufferOrEvent());
}