Examples with BufferDecompressor org.apache.flink.runtime.io.network.buffer.BufferDecompressor used on opensource projects

Example 1 with BufferDecompressor

use of org.apache.flink.runtime.io.network.buffer.BufferDecompressor in project flink by apache.

the class SingleInputGateFactory method create.

/**
 * Creates an input gate and all of its input channels.
 */
public SingleInputGate create(@Nonnull ShuffleIOOwnerContext owner, int gateIndex, @Nonnull InputGateDeploymentDescriptor igdd, @Nonnull PartitionProducerStateProvider partitionProducerStateProvider) {
    SupplierWithException<BufferPool, IOException> bufferPoolFactory = createBufferPoolFactory(networkBufferPool, floatingNetworkBuffersPerGate);
    BufferDecompressor bufferDecompressor = null;
    if (igdd.getConsumedPartitionType().isBlocking() && blockingShuffleCompressionEnabled) {
        bufferDecompressor = new BufferDecompressor(networkBufferSize, compressionCodec);
    }
    final String owningTaskName = owner.getOwnerName();
    final MetricGroup networkInputGroup = owner.getInputGroup();
    SubpartitionIndexRange subpartitionIndexRange = igdd.getConsumedSubpartitionIndexRange();
    SingleInputGate inputGate = new SingleInputGate(owningTaskName, gateIndex, igdd.getConsumedResultId(), igdd.getConsumedPartitionType(), subpartitionIndexRange, calculateNumChannels(igdd.getShuffleDescriptors().length, subpartitionIndexRange), partitionProducerStateProvider, bufferPoolFactory, bufferDecompressor, networkBufferPool, networkBufferSize, new ThroughputCalculator(SystemClock.getInstance()), maybeCreateBufferDebloater(gateIndex, networkInputGroup.addGroup(gateIndex)));
    InputChannelMetrics metrics = new InputChannelMetrics(networkInputGroup, owner.getParentGroup());
    createInputChannels(owningTaskName, igdd, inputGate, subpartitionIndexRange, metrics);
    return inputGate;
}

Example 2 with BufferDecompressor

use of org.apache.flink.runtime.io.network.buffer.BufferDecompressor in project flink by apache.

the class SingleInputGateTest method testGetCompressedBuffer.

/**
 * Tests that the compressed buffer will be decompressed after calling {@link
 * SingleInputGate#getNext()}.
 */
@Test
public void testGetCompressedBuffer() throws Exception {
    int bufferSize = 1024;
    String compressionCodec = "LZ4";
    BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
    BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
    try (SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).build()) {
        TestInputChannel inputChannel = new TestInputChannel(inputGate, 0);
        MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(bufferSize);
        for (int i = 0; i < bufferSize; i += 8) {
            segment.putLongLittleEndian(i, i);
        }
        Buffer uncompressedBuffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE);
        uncompressedBuffer.setSize(bufferSize);
        Buffer compressedBuffer = compressor.compressToOriginalBuffer(uncompressedBuffer);
        assertTrue(compressedBuffer.isCompressed());
        inputChannel.read(compressedBuffer);
        inputGate.setInputChannels(inputChannel);
        inputGate.notifyChannelNonEmpty(inputChannel);
        Optional<BufferOrEvent> bufferOrEvent = inputGate.getNext();
        assertTrue(bufferOrEvent.isPresent());
        assertTrue(bufferOrEvent.get().isBuffer());
        ByteBuffer buffer = bufferOrEvent.get().getBuffer().getNioBufferReadable().order(ByteOrder.LITTLE_ENDIAN);
        for (int i = 0; i < bufferSize; i += 8) {
            assertEquals(i, buffer.getLong());
        }
    }
}

Example 3 with BufferDecompressor

use of org.apache.flink.runtime.io.network.buffer.BufferDecompressor in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReceiveCompressedBuffer.

/**
 * Verifies that {@link BufferResponse} of compressed {@link Buffer} can be handled correctly.
 */
@Test
public void testReceiveCompressedBuffer() throws Exception {
    int bufferSize = 1024;
    String compressionCodec = "LZ4";
    BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
    BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).setSegmentProvider(networkBufferPool).build();
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
    inputGate.setInputChannels(inputChannel);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
        handler.addInputChannel(inputChannel);
        Buffer buffer = compressor.compressToOriginalBuffer(TestBufferFactory.createBuffer(bufferSize));
        BufferResponse bufferResponse = createBufferResponse(buffer, 0, inputChannel.getInputChannelId(), 2, new NetworkBufferAllocator(handler));
        assertTrue(bufferResponse.isCompressed);
        handler.channelRead(null, bufferResponse);
        Buffer receivedBuffer = inputChannel.getNextReceivedBuffer();
        assertNotNull(receivedBuffer);
        assertTrue(receivedBuffer.isCompressed());
        receivedBuffer.recycleBuffer();
    } finally {
        releaseResource(inputGate, networkBufferPool);
    }
}