Search in sources :

Example 1 with BufferAndBacklog

use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.

the class SequentialChannelStateReaderImplTest method collectBuffers.

private Map<ResultSubpartitionInfo, List<Buffer>> collectBuffers(BufferWritingResultPartition[] resultPartitions) throws IOException {
    Map<ResultSubpartitionInfo, List<Buffer>> actual = new HashMap<>();
    for (BufferWritingResultPartition resultPartition : resultPartitions) {
        for (int i = 0; i < resultPartition.getNumberOfSubpartitions(); i++) {
            ResultSubpartitionInfo info = resultPartition.getAllPartitions()[i].getSubpartitionInfo();
            ResultSubpartitionView view = resultPartition.createSubpartitionView(info.getSubPartitionIdx(), new NoOpBufferAvailablityListener());
            for (BufferAndBacklog buffer = view.getNextBuffer(); buffer != null; buffer = view.getNextBuffer()) {
                if (buffer.buffer().isBuffer()) {
                    actual.computeIfAbsent(info, unused -> new ArrayList<>()).add(buffer.buffer());
                }
            }
        }
    }
    return actual;
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog)

Example 2 with BufferAndBacklog

use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.

the class FileChannelBoundedDataTest method testRecycleBufferForNotifyingBufferAvailabilityListener.

@Test
public void testRecycleBufferForNotifyingBufferAvailabilityListener() throws Exception {
    final ResultSubpartition subpartition = createFileBoundedBlockingSubpartition();
    final int numberOfBuffers = 2;
    writeBuffers(subpartition, numberOfBuffers);
    final VerifyNotificationBufferAvailabilityListener listener = new VerifyNotificationBufferAvailabilityListener();
    final ResultSubpartitionView subpartitionView = createView(subpartition, listener);
    assertFalse(listener.isAvailable);
    final BufferAndBacklog buffer1 = subpartitionView.getNextBuffer();
    final BufferAndBacklog buffer2 = subpartitionView.getNextBuffer();
    assertNotNull(buffer1);
    assertNotNull(buffer2);
    // the next buffer is null in view because FileBufferReader has no available buffers for
    // reading ahead
    assertFalse(subpartitionView.getAvailabilityAndBacklog(Integer.MAX_VALUE).isAvailable());
    // recycle a buffer to trigger notification of data available
    buffer1.buffer().recycleBuffer();
    assertTrue(listener.isAvailable);
    // cleanup
    buffer2.buffer().recycleBuffer();
    subpartitionView.releaseAllResources();
    subpartition.release();
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) Test(org.junit.Test)

Example 3 with BufferAndBacklog

use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.

the class BoundedBlockingSubpartitionWriteReadTest method readLongs.

// ------------------------------------------------------------------------
// common test passes
// ------------------------------------------------------------------------
private static void readLongs(ResultSubpartitionView reader, long numLongs, int numBuffers, boolean compressionEnabled, BufferDecompressor decompressor) throws Exception {
    BufferAndBacklog next;
    long expectedNextLong = 0L;
    int nextExpectedBacklog = numBuffers - 1;
    while ((next = reader.getNextBuffer()) != null && next.buffer().isBuffer()) {
        assertTrue(next.isDataAvailable());
        assertEquals(nextExpectedBacklog, next.buffersInBacklog());
        ByteBuffer buffer = next.buffer().getNioBufferReadable();
        if (compressionEnabled && next.buffer().isCompressed()) {
            Buffer uncompressedBuffer = decompressor.decompressToIntermediateBuffer(next.buffer());
            buffer = uncompressedBuffer.getNioBufferReadable();
            uncompressedBuffer.recycleBuffer();
        }
        while (buffer.hasRemaining()) {
            assertEquals(expectedNextLong++, buffer.getLong());
        }
        next.buffer().recycleBuffer();
        nextExpectedBacklog--;
    }
    assertEquals(numLongs, expectedNextLong);
    assertEquals(-1, nextExpectedBacklog);
}
Also used : NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) ByteBuffer(java.nio.ByteBuffer) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteBuffer(java.nio.ByteBuffer)

Example 4 with BufferAndBacklog

use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.

the class TestSubpartitionConsumer method call.

@Override
public Boolean call() throws Exception {
    try {
        while (true) {
            if (Thread.interrupted()) {
                throw new InterruptedException();
            }
            synchronized (dataAvailableNotification) {
                while (!dataAvailableNotification.getAndSet(false)) {
                    dataAvailableNotification.wait();
                }
            }
            final BufferAndBacklog bufferAndBacklog = subpartitionView.getNextBuffer();
            if (isSlowConsumer) {
                Thread.sleep(random.nextInt(MAX_SLEEP_TIME_MS + 1));
            }
            if (bufferAndBacklog != null) {
                if (bufferAndBacklog.isDataAvailable()) {
                    dataAvailableNotification.set(true);
                }
                if (bufferAndBacklog.buffer().isBuffer()) {
                    callback.onBuffer(bufferAndBacklog.buffer());
                } else {
                    final AbstractEvent event = EventSerializer.fromBuffer(bufferAndBacklog.buffer(), getClass().getClassLoader());
                    callback.onEvent(event);
                    bufferAndBacklog.buffer().recycleBuffer();
                    if (event.getClass() == EndOfPartitionEvent.class) {
                        subpartitionView.releaseAllResources();
                        return true;
                    }
                }
            } else if (subpartitionView.isReleased()) {
                return true;
            }
        }
    } finally {
        subpartitionView.releaseAllResources();
    }
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) AbstractEvent(org.apache.flink.runtime.event.AbstractEvent)

Example 5 with BufferAndBacklog

use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.

the class LocalInputChannel method getNextBuffer.

@Override
Optional<BufferAndAvailability> getNextBuffer() throws IOException {
    checkError();
    ResultSubpartitionView subpartitionView = this.subpartitionView;
    if (subpartitionView == null) {
        // during) it was released during reading the EndOfPartitionEvent (2).
        if (isReleased) {
            return Optional.empty();
        }
        // this can happen if the request for the partition was triggered asynchronously
        // by the time trigger
        // would be good to avoid that, by guaranteeing that the requestPartition() and
        // getNextBuffer() always come from the same thread
        // we could do that by letting the timer insert a special "requesting channel" into the
        // input gate's queue
        subpartitionView = checkAndWaitForSubpartitionView();
    }
    BufferAndBacklog next = subpartitionView.getNextBuffer();
    // ignore the empty buffer directly
    while (next != null && next.buffer().readableBytes() == 0) {
        next.buffer().recycleBuffer();
        next = subpartitionView.getNextBuffer();
        numBuffersIn.inc();
    }
    if (next == null) {
        if (subpartitionView.isReleased()) {
            throw new CancelTaskException("Consumed partition " + subpartitionView + " has been released.");
        } else {
            return Optional.empty();
        }
    }
    Buffer buffer = next.buffer();
    if (buffer instanceof FileRegionBuffer) {
        buffer = ((FileRegionBuffer) buffer).readInto(inputGate.getUnpooledSegment());
    }
    numBytesIn.inc(buffer.getSize());
    numBuffersIn.inc();
    channelStatePersister.checkForBarrier(buffer);
    channelStatePersister.maybePersist(buffer);
    NetworkActionsLogger.traceInput("LocalInputChannel#getNextBuffer", buffer, inputGate.getOwningTaskName(), channelInfo, channelStatePersister, next.getSequenceNumber());
    return Optional.of(new BufferAndAvailability(buffer, next.getNextDataType(), next.buffersInBacklog(), next.getSequenceNumber()));
}
Also used : FileRegionBuffer(org.apache.flink.runtime.io.network.buffer.FileRegionBuffer) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) FileRegionBuffer(org.apache.flink.runtime.io.network.buffer.FileRegionBuffer) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog)

Aggregations

BufferAndBacklog (org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog)6 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)3 ArrayList (java.util.ArrayList)2 NetworkBuffer (org.apache.flink.runtime.io.network.buffer.NetworkBuffer)2 ResultSubpartitionView (org.apache.flink.runtime.io.network.partition.ResultSubpartitionView)2 Test (org.junit.Test)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 DataOutputStream (java.io.DataOutputStream)1 IOException (java.io.IOException)1 ByteBuffer (java.nio.ByteBuffer)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Random (java.util.Random)1 BiFunction (java.util.function.BiFunction)1 Function (java.util.function.Function)1 Function.identity (java.util.function.Function.identity)1 Collectors.toList (java.util.stream.Collectors.toList)1