Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 61 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordOrEventCollectingResultPartitionWriter method broadcastEvent.

@Override
public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
    // visible in ITCases or end to end tests.
    try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event, isPriorityEvent)) {
        Buffer buffer = eventBufferConsumer.build();
        try {
            AbstractEvent deserializedEvent = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
            output.add(deserializedEvent);
        } finally {
            buffer.recycleBuffer();
        }
    }
}

Example 62 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriterTest method verifyDeserializationResults.

protected void verifyDeserializationResults(ResultSubpartitionView view, RecordDeserializer<SerializationTestType> deserializer, ArrayDeque<SerializationTestType> expectedRecords, int numRequiredBuffers, int numValues) throws Exception {
    int assertRecords = 0;
    for (int j = 0; j < numRequiredBuffers; j++) {
        Buffer buffer = view.getNextBuffer().buffer();
        deserializer.setNextBuffer(buffer);
        assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer);
    }
    assertFalse(view.getAvailabilityAndBacklog(Integer.MAX_VALUE).isAvailable());
    Assert.assertEquals(numValues, assertRecords);
}

Example 63 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriterTest method testIsAvailableOrNot.

/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one
 * available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
    // setup
    final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128);
    final BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    final ResultPartitionWriter resultPartition = new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    resultPartition.setup();
    final RecordWriter<?> recordWriter = createRecordWriter(resultPartition);
    try {
        // record writer is available because of initial available global pool
        assertTrue(recordWriter.getAvailableFuture().isDone());
        // request one buffer from the local pool to make it unavailable afterwards
        try (BufferBuilder bufferBuilder = localPool.requestBufferBuilder(0)) {
            assertNotNull(bufferBuilder);
            assertFalse(recordWriter.getAvailableFuture().isDone());
            // recycle the buffer to make the local pool available again
            final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
            buffer.recycleBuffer();
        }
        assertTrue(recordWriter.getAvailableFuture().isDone());
        assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());
    } finally {
        localPool.lazyDestroy();
        globalPool.destroy();
    }
}

Example 64 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriterTest method testBroadcastEventBufferReferenceCounting.

/**
 * Tests that event buffers are properly recycled when broadcasting events to multiple channels.
 */
@Test
public void testBroadcastEventBufferReferenceCounting() throws Exception {
    int bufferSize = 32 * 1024;
    int numSubpartitions = 2;
    ResultPartition partition = createResultPartition(bufferSize, numSubpartitions);
    RecordWriter<?> writer = createRecordWriter(partition);
    writer.broadcastEvent(EndOfPartitionEvent.INSTANCE);
    // get references to buffer consumers (copies from the original event buffer consumer)
    Buffer[] buffers = new Buffer[numSubpartitions];
    // process all collected events (recycles the buffer)
    for (int i = 0; i < numSubpartitions; i++) {
        assertEquals(1, partition.getNumberOfQueuedBuffers(i));
        ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
        buffers[i] = view.getNextBuffer().buffer();
        assertTrue(parseBuffer(buffers[i], i).isEvent());
    }
    for (int i = 0; i < numSubpartitions; ++i) {
        assertTrue(buffers[i].isRecycled());
    }
}

Example 65 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriterTest method verifyBroadcastBufferOrEventIndependence.

private void verifyBroadcastBufferOrEventIndependence(boolean broadcastEvent) throws Exception {
    ResultPartition partition = createResultPartition(4096, 2);
    RecordWriter<IntValue> writer = createRecordWriter(partition);
    if (broadcastEvent) {
        writer.broadcastEvent(EndOfPartitionEvent.INSTANCE);
    } else {
        writer.broadcastEmit(new IntValue(0));
    }
    // verify added to all queues
    assertEquals(1, partition.getNumberOfQueuedBuffers(0));
    assertEquals(1, partition.getNumberOfQueuedBuffers(1));
    ResultSubpartitionView view0 = partition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
    ResultSubpartitionView view1 = partition.createSubpartitionView(1, new NoOpBufferAvailablityListener());
    // these two buffers may share the memory but not the indices!
    Buffer buffer1 = view0.getNextBuffer().buffer();
    Buffer buffer2 = view1.getNextBuffer().buffer();
    assertEquals(0, buffer1.getReaderIndex());
    assertEquals(0, buffer2.getReaderIndex());
    buffer1.setReaderIndex(1);
    assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex());
}