Examples with CompositeByteBuf org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf used on opensource projects

Example 61 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class RetainingAsyncWritableChannel method writeInternal.

/**
 * Internal method for writing to the channel that allows for a pluggable action to be taken to produce a
 * retained {@link ByteBuf} that can be added to the composite buffer.
 * @param retainedBufSupplier creates a retained {@link ByteBuf} that can be freely added to a
 *                            {@link CompositeByteBuf}.
 * @param callback called once the buffer has been added.
 * @return a future that is completed once the buffer has been added.
 */
private Future<Long> writeInternal(Supplier<ByteBuf> retainedBufSupplier, Callback<Long> callback) {
    FutureResult<Long> future = new FutureResult<>();
    ByteBuf buf = null;
    long bytesWritten = 0;
    Exception exception = null;
    try {
        if (!isOpen()) {
            throw new ClosedChannelException();
        } else if (totalBytesWritten.get() > sizeLimitInBytes) {
            throw new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
        } else {
            synchronized (bufferLock) {
                if (compositeBuffer == null) {
                    throw new IllegalStateException("Cannot write more data; content already consumed or channel was closed");
                }
                buf = retainedBufSupplier.get();
                bytesWritten = buf.readableBytes();
                compositeBuffer.addComponent(true, buf);
            }
            if (totalBytesWritten.addAndGet(bytesWritten) > sizeLimitInBytes) {
                exception = new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
            }
        }
    } catch (Exception e) {
        exception = e;
        if (buf != null) {
            buf.release();
        }
    } finally {
        future.done(bytesWritten, exception);
        if (callback != null) {
            callback.onCompletion(bytesWritten, exception);
        }
    }
    return future;
}

Example 62 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class GetResponse method prepareBuffer.

/**
 * A private method shared by {@link GetResponse#writeTo(WritableByteChannel)} and
 * {@link GetResponse#writeTo(AsyncWritableChannel, Callback)}.
 * This method allocate bufferToSend and write metadata to it if bufferToSend is null.
 */
@Override
protected void prepareBuffer() {
    bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer((int) super.sizeInBytes() + (Partition_Response_Info_List_Size + partitionResponseInfoSize));
    writeHeader();
    if (partitionResponseInfoList != null) {
        bufferToSend.writeInt(partitionResponseInfoList.size());
        for (PartitionResponseInfo partitionResponseInfo : partitionResponseInfoList) {
            partitionResponseInfo.writeTo(bufferToSend);
        }
    }
    if (toSend != null) {
        ByteBuf toSendContent = toSend.content();
        if (toSendContent != null) {
            // Since this composite blob will be a readonly blob, we don't really care about if it's allocated
            // on a direct memory or not.
            CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeDirectBuffer();
            int maxNumComponent = 1 + toSendContent.nioBufferCount();
            compositeByteBuf.addComponent(true, bufferToSend);
            compositeByteBuf.addComponent(true, toSendContent);
            bufferToSend = compositeByteBuf;
            sendSizeInBufferToSend = toSendContent.readableBytes();
            toSend = null;
        }
    }
}

Example 63 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class PutRequest method prepareBuffer.

/**
 * Construct the bufferToSend to serialize request metadata and other blob related information. The newly constructed
 * bufferToSend will not include the blob content as it's carried by the {@code blob} field in this class.
 */
@Override
protected void prepareBuffer() {
    // bufferToSend now is the header ByteBuf, it will store serialized header, without blob content and crc
    bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer(sizeExcludingBlobAndCrcSize());
    writeHeader();
    int crcStart = bufferToSend.writerIndex();
    bufferToSend.writeBytes(blobId.toBytes());
    BlobPropertiesSerDe.serializeBlobProperties(bufferToSend, properties);
    bufferToSend.writeInt(usermetadata.capacity());
    bufferToSend.writeBytes(usermetadata);
    bufferToSend.writeShort((short) blobType.ordinal());
    short keyLength = blobEncryptionKey == null ? 0 : (short) blobEncryptionKey.remaining();
    bufferToSend.writeShort(keyLength);
    if (keyLength > 0) {
        bufferToSend.writeBytes(blobEncryptionKey);
    }
    bufferToSend.writeLong(blobSize);
    // Now compute crc for the put request.
    crc.update(bufferToSend.nioBuffer(crcStart, bufferToSend.writerIndex() - crcStart));
    for (ByteBuffer bb : blob.nioBuffers()) {
        crc.update(bb);
        // change it back to 0 since we are going to write it to the channel later.
        bb.position(0);
    }
    crcByteBuf = PooledByteBufAllocator.DEFAULT.ioBuffer(CRC_SIZE_IN_BYTES);
    crcByteBuf.writeLong(crc.getValue());
    // Now construct the real bufferToSend, which should be a composite ByteBuf.
    CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeHeapBuffer(2 + blob.nioBufferCount());
    compositeByteBuf.addComponent(true, bufferToSend);
    if (blob instanceof CompositeByteBuf) {
        Iterator<ByteBuf> iter = ((CompositeByteBuf) blob).iterator();
        while (iter.hasNext()) {
            compositeByteBuf.addComponent(true, iter.next());
        }
    } else {
        compositeByteBuf.addComponent(true, blob);
    }
    compositeByteBuf.addComponent(true, crcByteBuf);
    blob = null;
    crcByteBuf = null;
    bufferToSend = compositeByteBuf;
}

Example 64 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class CopyForcingByteBuf method generateCompositeContent.

/**
 * Generates random content and fills it up in {@code httpContents} with a backing {@link CompositeByteBuf}.
 * @param httpContents the {@link List<HttpContent>} that will contain all the content.
 * @return the whole content as a {@link ByteBuffer} - serves as a source of truth.
 */
private ByteBuffer generateCompositeContent(List<HttpContent> httpContents) {
    int individualPartSize = GENERATED_CONTENT_SIZE / GENERATED_CONTENT_PART_COUNT;
    byte[] contentBytes = TestUtils.getRandomBytes(GENERATED_CONTENT_SIZE);
    ArrayList<ByteBuf> byteBufs = new ArrayList<>(GENERATED_CONTENT_PART_COUNT);
    for (int addedContentCount = 0; addedContentCount < GENERATED_CONTENT_PART_COUNT; addedContentCount++) {
        byteBufs.add(Unpooled.wrappedBuffer(contentBytes, addedContentCount * individualPartSize, individualPartSize));
    }
    httpContents.add(new DefaultLastHttpContent(new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 20, byteBufs)));
    return ByteBuffer.wrap(contentBytes);
}

Example 65 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class GCMCryptoServiceTest method testEncryptDecryptNettyCompositeByteBuf.

@Test
public void testEncryptDecryptNettyCompositeByteBuf() throws Exception {
    // testEncryptDecryptNettyByte already tests the correctness of the encrypt decrypt methods with non-composite Netty
    // ByteBuf, in this test case, we can make the assumption that these two functions always provide correct answers.
    String key = TestUtils.getRandomKey(DEFAULT_KEY_SIZE_IN_CHARS);
    Properties props = getKMSProperties(key, DEFAULT_KEY_SIZE_IN_CHARS);
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    SecretKeySpec secretKeySpec = new SecretKeySpec(Hex.decode(key), "AES");
    GCMCryptoService cryptoService = (GCMCryptoService) (new GCMCryptoServiceFactory(verifiableProperties, REGISTRY).getCryptoService());
    byte[] fixedIv = new byte[12];
    for (int i = 0; i < 5; i++) {
        int size = TestUtils.RANDOM.nextInt(MAX_DATA_SIZE - MIN_DATA_SIZE) + MIN_DATA_SIZE;
        byte[] randomData = new byte[size];
        TestUtils.RANDOM.nextBytes(randomData);
        ByteBuf toEncrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(randomData.length);
        toEncrypt.writeBytes(randomData);
        CompositeByteBuf toEncryptComposite = toEncrypt.alloc().compositeBuffer(size);
        int start = 0;
        int end = 0;
        for (int j = 0; j < 3; j++) {
            start = end;
            end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
            if (j == 2) {
                end = size;
            }
            ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
            c.writeBytes(randomData, start, end - start);
            toEncryptComposite.addComponent(true, c);
        }
        ByteBuf encryptedBytes = cryptoService.encrypt(toEncrypt, secretKeySpec, fixedIv);
        ByteBuf encryptedBytesComposite = cryptoService.encrypt(toEncryptComposite, secretKeySpec, fixedIv);
        Assert.assertEquals(encryptedBytes.readableBytes(), encryptedBytesComposite.readableBytes());
        Assert.assertEquals(toEncrypt.readableBytes(), 0);
        Assert.assertEquals(toEncryptComposite.readableBytes(), 0);
        byte[] array = new byte[encryptedBytes.readableBytes()];
        encryptedBytes.getBytes(encryptedBytes.readerIndex(), array);
        byte[] arrayComposite = new byte[encryptedBytesComposite.readableBytes()];
        encryptedBytesComposite.getBytes(encryptedBytesComposite.readerIndex(), arrayComposite);
        Assert.assertArrayEquals(array, arrayComposite);
        toEncrypt.release();
        toEncryptComposite.release();
        encryptedBytes.release();
        encryptedBytesComposite.release();
        ByteBuf toDecrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(array.length);
        toDecrypt.writeBytes(array);
        CompositeByteBuf toDecryptComposite = toDecrypt.alloc().compositeBuffer(size);
        size = array.length;
        start = 0;
        end = 0;
        for (int j = 0; j < 3; j++) {
            start = end;
            end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
            if (j == 2) {
                end = size;
            }
            ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
            c.writeBytes(array, start, end - start);
            toDecryptComposite.addComponent(true, c);
        }
        ByteBuf decryptedBytes = cryptoService.decrypt(toDecrypt, secretKeySpec);
        ByteBuf decryptedBytesComposite = cryptoService.decrypt(toDecryptComposite, secretKeySpec);
        Assert.assertEquals(decryptedBytes.readableBytes(), decryptedBytesComposite.readableBytes());
        Assert.assertEquals(toDecrypt.readableBytes(), 0);
        Assert.assertEquals(toDecryptComposite.readableBytes(), 0);
        array = new byte[decryptedBytes.readableBytes()];
        arrayComposite = new byte[decryptedBytesComposite.readableBytes()];
        decryptedBytes.getBytes(decryptedBytes.readerIndex(), array);
        decryptedBytesComposite.getBytes(decryptedBytesComposite.readerIndex(), arrayComposite);
        Assert.assertArrayEquals(array, arrayComposite);
        toDecrypt.release();
        toDecryptComposite.release();
        decryptedBytes.release();
        decryptedBytesComposite.release();
    }
}