use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.
the class SequentialChannelStateReaderImplTest method collectBuffers.
private Map<ResultSubpartitionInfo, List<Buffer>> collectBuffers(BufferWritingResultPartition[] resultPartitions) throws IOException {
Map<ResultSubpartitionInfo, List<Buffer>> actual = new HashMap<>();
for (BufferWritingResultPartition resultPartition : resultPartitions) {
for (int i = 0; i < resultPartition.getNumberOfSubpartitions(); i++) {
ResultSubpartitionInfo info = resultPartition.getAllPartitions()[i].getSubpartitionInfo();
ResultSubpartitionView view = resultPartition.createSubpartitionView(info.getSubPartitionIdx(), new NoOpBufferAvailablityListener());
for (BufferAndBacklog buffer = view.getNextBuffer(); buffer != null; buffer = view.getNextBuffer()) {
if (buffer.buffer().isBuffer()) {
actual.computeIfAbsent(info, unused -> new ArrayList<>()).add(buffer.buffer());
}
}
}
}
return actual;
}
use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.
the class FileChannelBoundedDataTest method testRecycleBufferForNotifyingBufferAvailabilityListener.
@Test
public void testRecycleBufferForNotifyingBufferAvailabilityListener() throws Exception {
final ResultSubpartition subpartition = createFileBoundedBlockingSubpartition();
final int numberOfBuffers = 2;
writeBuffers(subpartition, numberOfBuffers);
final VerifyNotificationBufferAvailabilityListener listener = new VerifyNotificationBufferAvailabilityListener();
final ResultSubpartitionView subpartitionView = createView(subpartition, listener);
assertFalse(listener.isAvailable);
final BufferAndBacklog buffer1 = subpartitionView.getNextBuffer();
final BufferAndBacklog buffer2 = subpartitionView.getNextBuffer();
assertNotNull(buffer1);
assertNotNull(buffer2);
// the next buffer is null in view because FileBufferReader has no available buffers for
// reading ahead
assertFalse(subpartitionView.getAvailabilityAndBacklog(Integer.MAX_VALUE).isAvailable());
// recycle a buffer to trigger notification of data available
buffer1.buffer().recycleBuffer();
assertTrue(listener.isAvailable);
// cleanup
buffer2.buffer().recycleBuffer();
subpartitionView.releaseAllResources();
subpartition.release();
}
use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.
the class BoundedBlockingSubpartitionWriteReadTest method readLongs.
// ------------------------------------------------------------------------
// common test passes
// ------------------------------------------------------------------------
private static void readLongs(ResultSubpartitionView reader, long numLongs, int numBuffers, boolean compressionEnabled, BufferDecompressor decompressor) throws Exception {
BufferAndBacklog next;
long expectedNextLong = 0L;
int nextExpectedBacklog = numBuffers - 1;
while ((next = reader.getNextBuffer()) != null && next.buffer().isBuffer()) {
assertTrue(next.isDataAvailable());
assertEquals(nextExpectedBacklog, next.buffersInBacklog());
ByteBuffer buffer = next.buffer().getNioBufferReadable();
if (compressionEnabled && next.buffer().isCompressed()) {
Buffer uncompressedBuffer = decompressor.decompressToIntermediateBuffer(next.buffer());
buffer = uncompressedBuffer.getNioBufferReadable();
uncompressedBuffer.recycleBuffer();
}
while (buffer.hasRemaining()) {
assertEquals(expectedNextLong++, buffer.getLong());
}
next.buffer().recycleBuffer();
nextExpectedBacklog--;
}
assertEquals(numLongs, expectedNextLong);
assertEquals(-1, nextExpectedBacklog);
}
use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.
the class TestSubpartitionConsumer method call.
@Override
public Boolean call() throws Exception {
try {
while (true) {
if (Thread.interrupted()) {
throw new InterruptedException();
}
synchronized (dataAvailableNotification) {
while (!dataAvailableNotification.getAndSet(false)) {
dataAvailableNotification.wait();
}
}
final BufferAndBacklog bufferAndBacklog = subpartitionView.getNextBuffer();
if (isSlowConsumer) {
Thread.sleep(random.nextInt(MAX_SLEEP_TIME_MS + 1));
}
if (bufferAndBacklog != null) {
if (bufferAndBacklog.isDataAvailable()) {
dataAvailableNotification.set(true);
}
if (bufferAndBacklog.buffer().isBuffer()) {
callback.onBuffer(bufferAndBacklog.buffer());
} else {
final AbstractEvent event = EventSerializer.fromBuffer(bufferAndBacklog.buffer(), getClass().getClassLoader());
callback.onEvent(event);
bufferAndBacklog.buffer().recycleBuffer();
if (event.getClass() == EndOfPartitionEvent.class) {
subpartitionView.releaseAllResources();
return true;
}
}
} else if (subpartitionView.isReleased()) {
return true;
}
}
} finally {
subpartitionView.releaseAllResources();
}
}
use of org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog in project flink by apache.
the class LocalInputChannel method getNextBuffer.
@Override
Optional<BufferAndAvailability> getNextBuffer() throws IOException {
checkError();
ResultSubpartitionView subpartitionView = this.subpartitionView;
if (subpartitionView == null) {
// during) it was released during reading the EndOfPartitionEvent (2).
if (isReleased) {
return Optional.empty();
}
// this can happen if the request for the partition was triggered asynchronously
// by the time trigger
// would be good to avoid that, by guaranteeing that the requestPartition() and
// getNextBuffer() always come from the same thread
// we could do that by letting the timer insert a special "requesting channel" into the
// input gate's queue
subpartitionView = checkAndWaitForSubpartitionView();
}
BufferAndBacklog next = subpartitionView.getNextBuffer();
// ignore the empty buffer directly
while (next != null && next.buffer().readableBytes() == 0) {
next.buffer().recycleBuffer();
next = subpartitionView.getNextBuffer();
numBuffersIn.inc();
}
if (next == null) {
if (subpartitionView.isReleased()) {
throw new CancelTaskException("Consumed partition " + subpartitionView + " has been released.");
} else {
return Optional.empty();
}
}
Buffer buffer = next.buffer();
if (buffer instanceof FileRegionBuffer) {
buffer = ((FileRegionBuffer) buffer).readInto(inputGate.getUnpooledSegment());
}
numBytesIn.inc(buffer.getSize());
numBuffersIn.inc();
channelStatePersister.checkForBarrier(buffer);
channelStatePersister.maybePersist(buffer);
NetworkActionsLogger.traceInput("LocalInputChannel#getNextBuffer", buffer, inputGate.getOwningTaskName(), channelInfo, channelStatePersister, next.getSequenceNumber());
return Optional.of(new BufferAndAvailability(buffer, next.getNextDataType(), next.buffersInBacklog(), next.getSequenceNumber()));
}
Aggregations