Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 51 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CheckpointedInputGateTest method testUpstreamResumedUponEndOfRecovery.

@Test
public void testUpstreamResumedUponEndOfRecovery() throws Exception {
    int numberOfChannels = 11;
    NetworkBufferPool bufferPool = new NetworkBufferPool(numberOfChannels * 3, 1024);
    try {
        ResumeCountingConnectionManager resumeCounter = new ResumeCountingConnectionManager();
        CheckpointedInputGate gate = setupInputGate(numberOfChannels, bufferPool, resumeCounter);
        assertFalse(gate.pollNext().isPresent());
        for (int channelIndex = 0; channelIndex < numberOfChannels - 1; channelIndex++) {
            enqueueEndOfState(gate, channelIndex);
            Optional<BufferOrEvent> bufferOrEvent = gate.pollNext();
            while (bufferOrEvent.isPresent() && bufferOrEvent.get().getEvent() instanceof EndOfChannelStateEvent && !gate.allChannelsRecovered()) {
                bufferOrEvent = gate.pollNext();
            }
            assertFalse("should align (block all channels)", bufferOrEvent.isPresent());
        }
        enqueueEndOfState(gate, numberOfChannels - 1);
        Optional<BufferOrEvent> polled = gate.pollNext();
        assertTrue(polled.isPresent());
        assertTrue(polled.get().isEvent());
        assertEquals(EndOfChannelStateEvent.INSTANCE, polled.get().getEvent());
        assertEquals(numberOfChannels, resumeCounter.getNumResumed());
        assertFalse("should only be a single event no matter of what is the number of channels", gate.pollNext().isPresent());
    } finally {
        bufferPool.destroy();
    }
}

Example 52 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CheckpointedInputGateTest method testPersisting.

/**
 * This tests a scenario where an older triggered checkpoint, was cancelled and a newer
 * checkpoint was triggered very quickly after the cancellation. It can happen that a task can
 * receive first the more recent checkpoint barrier and later the obsoleted one. This can happen
 * for many reasons (for example Source tasks not running, or just a race condition with
 * notifyCheckpointAborted RPCs) and Task should be able to handle this properly. In FLINK-21104
 * the problem was that this obsoleted checkpoint barrier was causing a checkState to fail.
 */
public void testPersisting(boolean drainGate) throws Exception {
    int numberOfChannels = 3;
    NetworkBufferPool bufferPool = new NetworkBufferPool(numberOfChannels * 3, 1024);
    try {
        long checkpointId = 2L;
        long obsoleteCheckpointId = 1L;
        ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(checkpointId);
        RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
        CheckpointedInputGate gate = setupInputGateWithAlternatingController(numberOfChannels, bufferPool, validatingHandler, stateWriter);
        // enqueue first checkpointId before obsoleteCheckpointId, so that we never trigger
        // and also never cancel the obsoleteCheckpointId
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 0, barrier(checkpointId));
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 1, barrier(obsoleteCheckpointId));
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 2, buildSomeBuffer());
        assertEquals(0, validatingHandler.getTriggeredCheckpointCounter());
        // trigger checkpoint
        gate.pollNext();
        assertEquals(1, validatingHandler.getTriggeredCheckpointCounter());
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 2);
        assertAddedInputSize(stateWriter, 2, 1);
        enqueue(gate, 0, buildSomeBuffer());
        enqueue(gate, 1, buildSomeBuffer());
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 2);
        enqueue(gate, 1, barrier(checkpointId));
        enqueue(gate, 1, buildSomeBuffer());
        // Another obsoleted barrier that should be ignored
        enqueue(gate, 2, barrier(obsoleteCheckpointId));
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 3);
        enqueue(gate, 2, barrier(checkpointId));
        enqueue(gate, 2, buildSomeBuffer());
        while (drainGate && gate.pollNext().isPresent()) {
        }
        assertAddedInputSize(stateWriter, 0, 1);
        assertAddedInputSize(stateWriter, 1, 3);
        assertAddedInputSize(stateWriter, 2, 3);
    } finally {
        bufferPool.destroy();
    }
}

Example 53 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class TestCheckpointedInputGateBuilder method buildRemoteGate.

private SingleInputGate buildRemoteGate() throws IOException {
    int maxUsedBuffers = 10;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(numChannels * maxUsedBuffers, 4096);
    SingleInputGate gate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildRemoteChannel).setNumberOfChannels(numChannels).setSegmentProvider(networkBufferPool).setBufferPoolFactory(networkBufferPool.createBufferPool(numChannels, maxUsedBuffers)).setChannelStateWriter(channelStateWriter).build();
    gate.setup();
    gate.requestPartitions();
    return gate;
}

Example 54 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReceiveCompressedBuffer.

/**
 * Verifies that {@link BufferResponse} of compressed {@link Buffer} can be handled correctly.
 */
@Test
public void testReceiveCompressedBuffer() throws Exception {
    int bufferSize = 1024;
    String compressionCodec = "LZ4";
    BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
    BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).setSegmentProvider(networkBufferPool).build();
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
    inputGate.setInputChannels(inputChannel);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
        handler.addInputChannel(inputChannel);
        Buffer buffer = compressor.compressToOriginalBuffer(TestBufferFactory.createBuffer(bufferSize));
        BufferResponse bufferResponse = createBufferResponse(buffer, 0, inputChannel.getInputChannelId(), 2, new NetworkBufferAllocator(handler));
        assertTrue(bufferResponse.isCompressed);
        handler.channelRead(null, bufferResponse);
        Buffer receivedBuffer = inputChannel.getNextReceivedBuffer();
        assertNotNull(receivedBuffer);
        assertTrue(receivedBuffer.isCompressed());
        receivedBuffer.recycleBuffer();
    } finally {
        releaseResource(inputGate, networkBufferPool);
    }
}

Example 55 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testDoNotFailHandlerOnSingleChannelFailure.

@Test
public void testDoNotFailHandlerOnSingleChannelFailure() throws Exception {
    // Setup
    final int bufferSize = 1024;
    final String expectedMessage = "test exception on buffer";
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = new TestRemoteInputChannelForError(inputGate, expectedMessage);
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    try {
        inputGate.setInputChannels(inputChannel);
        inputGate.setup();
        inputGate.requestPartitions();
        handler.addInputChannel(inputChannel);
        final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(bufferSize), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        // It will trigger an expected exception from TestRemoteInputChannelForError#onBuffer
        handler.channelRead(null, bufferResponse);
        // The handler should not be tagged as error for above excepted exception
        handler.checkError();
        try {
            // The input channel should be tagged as error and the respective exception is
            // thrown via #getNext
            inputGate.getNext();
        } catch (IOException ignored) {
            assertEquals(expectedMessage, ignored.getMessage());
        }
    } finally {
        // Cleanup
        releaseResource(inputGate, networkBufferPool);
    }
}