Search in sources :

Example 61 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class RetainingAsyncWritableChannel method writeInternal.

/**
 * Internal method for writing to the channel that allows for a pluggable action to be taken to produce a
 * retained {@link ByteBuf} that can be added to the composite buffer.
 * @param retainedBufSupplier creates a retained {@link ByteBuf} that can be freely added to a
 *                            {@link CompositeByteBuf}.
 * @param callback called once the buffer has been added.
 * @return a future that is completed once the buffer has been added.
 */
private Future<Long> writeInternal(Supplier<ByteBuf> retainedBufSupplier, Callback<Long> callback) {
    FutureResult<Long> future = new FutureResult<>();
    ByteBuf buf = null;
    long bytesWritten = 0;
    Exception exception = null;
    try {
        if (!isOpen()) {
            throw new ClosedChannelException();
        } else if (totalBytesWritten.get() > sizeLimitInBytes) {
            throw new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
        } else {
            synchronized (bufferLock) {
                if (compositeBuffer == null) {
                    throw new IllegalStateException("Cannot write more data; content already consumed or channel was closed");
                }
                buf = retainedBufSupplier.get();
                bytesWritten = buf.readableBytes();
                compositeBuffer.addComponent(true, buf);
            }
            if (totalBytesWritten.addAndGet(bytesWritten) > sizeLimitInBytes) {
                exception = new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
            }
        }
    } catch (Exception e) {
        exception = e;
        if (buf != null) {
            buf.release();
        }
    } finally {
        future.done(bytesWritten, exception);
        if (callback != null) {
            callback.onCompletion(bytesWritten, exception);
        }
    }
    return future;
}
Also used : RestServiceException(com.github.ambry.rest.RestServiceException) ClosedChannelException(java.nio.channels.ClosedChannelException) FutureResult(com.github.ambry.router.FutureResult) AtomicLong(java.util.concurrent.atomic.AtomicLong) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) ClosedChannelException(java.nio.channels.ClosedChannelException) RestServiceException(com.github.ambry.rest.RestServiceException)

Example 62 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class GetResponse method prepareBuffer.

/**
 * A private method shared by {@link GetResponse#writeTo(WritableByteChannel)} and
 * {@link GetResponse#writeTo(AsyncWritableChannel, Callback)}.
 * This method allocate bufferToSend and write metadata to it if bufferToSend is null.
 */
@Override
protected void prepareBuffer() {
    bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer((int) super.sizeInBytes() + (Partition_Response_Info_List_Size + partitionResponseInfoSize));
    writeHeader();
    if (partitionResponseInfoList != null) {
        bufferToSend.writeInt(partitionResponseInfoList.size());
        for (PartitionResponseInfo partitionResponseInfo : partitionResponseInfoList) {
            partitionResponseInfo.writeTo(bufferToSend);
        }
    }
    if (toSend != null) {
        ByteBuf toSendContent = toSend.content();
        if (toSendContent != null) {
            // Since this composite blob will be a readonly blob, we don't really care about if it's allocated
            // on a direct memory or not.
            CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeDirectBuffer();
            int maxNumComponent = 1 + toSendContent.nioBufferCount();
            compositeByteBuf.addComponent(true, bufferToSend);
            compositeByteBuf.addComponent(true, toSendContent);
            bufferToSend = compositeByteBuf;
            sendSizeInBufferToSend = toSendContent.readableBytes();
            toSend = null;
        }
    }
}
Also used : CompositeByteBuf(io.netty.buffer.CompositeByteBuf) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf)

Example 63 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class PutRequest method prepareBuffer.

/**
 * Construct the bufferToSend to serialize request metadata and other blob related information. The newly constructed
 * bufferToSend will not include the blob content as it's carried by the {@code blob} field in this class.
 */
@Override
protected void prepareBuffer() {
    // bufferToSend now is the header ByteBuf, it will store serialized header, without blob content and crc
    bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer(sizeExcludingBlobAndCrcSize());
    writeHeader();
    int crcStart = bufferToSend.writerIndex();
    bufferToSend.writeBytes(blobId.toBytes());
    BlobPropertiesSerDe.serializeBlobProperties(bufferToSend, properties);
    bufferToSend.writeInt(usermetadata.capacity());
    bufferToSend.writeBytes(usermetadata);
    bufferToSend.writeShort((short) blobType.ordinal());
    short keyLength = blobEncryptionKey == null ? 0 : (short) blobEncryptionKey.remaining();
    bufferToSend.writeShort(keyLength);
    if (keyLength > 0) {
        bufferToSend.writeBytes(blobEncryptionKey);
    }
    bufferToSend.writeLong(blobSize);
    // Now compute crc for the put request.
    crc.update(bufferToSend.nioBuffer(crcStart, bufferToSend.writerIndex() - crcStart));
    for (ByteBuffer bb : blob.nioBuffers()) {
        crc.update(bb);
        // change it back to 0 since we are going to write it to the channel later.
        bb.position(0);
    }
    crcByteBuf = PooledByteBufAllocator.DEFAULT.ioBuffer(CRC_SIZE_IN_BYTES);
    crcByteBuf.writeLong(crc.getValue());
    // Now construct the real bufferToSend, which should be a composite ByteBuf.
    CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeHeapBuffer(2 + blob.nioBufferCount());
    compositeByteBuf.addComponent(true, bufferToSend);
    if (blob instanceof CompositeByteBuf) {
        Iterator<ByteBuf> iter = ((CompositeByteBuf) blob).iterator();
        while (iter.hasNext()) {
            compositeByteBuf.addComponent(true, iter.next());
        }
    } else {
        compositeByteBuf.addComponent(true, blob);
    }
    compositeByteBuf.addComponent(true, crcByteBuf);
    blob = null;
    crcByteBuf = null;
    bufferToSend = compositeByteBuf;
}
Also used : CompositeByteBuf(io.netty.buffer.CompositeByteBuf) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) ByteBuffer(java.nio.ByteBuffer)

Example 64 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class CopyForcingByteBuf method generateCompositeContent.

/**
 * Generates random content and fills it up in {@code httpContents} with a backing {@link CompositeByteBuf}.
 * @param httpContents the {@link List<HttpContent>} that will contain all the content.
 * @return the whole content as a {@link ByteBuffer} - serves as a source of truth.
 */
private ByteBuffer generateCompositeContent(List<HttpContent> httpContents) {
    int individualPartSize = GENERATED_CONTENT_SIZE / GENERATED_CONTENT_PART_COUNT;
    byte[] contentBytes = TestUtils.getRandomBytes(GENERATED_CONTENT_SIZE);
    ArrayList<ByteBuf> byteBufs = new ArrayList<>(GENERATED_CONTENT_PART_COUNT);
    for (int addedContentCount = 0; addedContentCount < GENERATED_CONTENT_PART_COUNT; addedContentCount++) {
        byteBufs.add(Unpooled.wrappedBuffer(contentBytes, addedContentCount * individualPartSize, individualPartSize));
    }
    httpContents.add(new DefaultLastHttpContent(new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 20, byteBufs)));
    return ByteBuffer.wrap(contentBytes);
}
Also used : CompositeByteBuf(io.netty.buffer.CompositeByteBuf) DefaultLastHttpContent(io.netty.handler.codec.http.DefaultLastHttpContent) ArrayList(java.util.ArrayList) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) UnpooledHeapByteBuf(io.netty.buffer.UnpooledHeapByteBuf) ByteBuf(io.netty.buffer.ByteBuf)

Example 65 with CompositeByteBuf

use of org.apache.flink.shaded.netty4.io.netty.buffer.CompositeByteBuf in project ambry by linkedin.

the class GCMCryptoServiceTest method testEncryptDecryptNettyCompositeByteBuf.

@Test
public void testEncryptDecryptNettyCompositeByteBuf() throws Exception {
    // testEncryptDecryptNettyByte already tests the correctness of the encrypt decrypt methods with non-composite Netty
    // ByteBuf, in this test case, we can make the assumption that these two functions always provide correct answers.
    String key = TestUtils.getRandomKey(DEFAULT_KEY_SIZE_IN_CHARS);
    Properties props = getKMSProperties(key, DEFAULT_KEY_SIZE_IN_CHARS);
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    SecretKeySpec secretKeySpec = new SecretKeySpec(Hex.decode(key), "AES");
    GCMCryptoService cryptoService = (GCMCryptoService) (new GCMCryptoServiceFactory(verifiableProperties, REGISTRY).getCryptoService());
    byte[] fixedIv = new byte[12];
    for (int i = 0; i < 5; i++) {
        int size = TestUtils.RANDOM.nextInt(MAX_DATA_SIZE - MIN_DATA_SIZE) + MIN_DATA_SIZE;
        byte[] randomData = new byte[size];
        TestUtils.RANDOM.nextBytes(randomData);
        ByteBuf toEncrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(randomData.length);
        toEncrypt.writeBytes(randomData);
        CompositeByteBuf toEncryptComposite = toEncrypt.alloc().compositeBuffer(size);
        int start = 0;
        int end = 0;
        for (int j = 0; j < 3; j++) {
            start = end;
            end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
            if (j == 2) {
                end = size;
            }
            ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
            c.writeBytes(randomData, start, end - start);
            toEncryptComposite.addComponent(true, c);
        }
        ByteBuf encryptedBytes = cryptoService.encrypt(toEncrypt, secretKeySpec, fixedIv);
        ByteBuf encryptedBytesComposite = cryptoService.encrypt(toEncryptComposite, secretKeySpec, fixedIv);
        Assert.assertEquals(encryptedBytes.readableBytes(), encryptedBytesComposite.readableBytes());
        Assert.assertEquals(toEncrypt.readableBytes(), 0);
        Assert.assertEquals(toEncryptComposite.readableBytes(), 0);
        byte[] array = new byte[encryptedBytes.readableBytes()];
        encryptedBytes.getBytes(encryptedBytes.readerIndex(), array);
        byte[] arrayComposite = new byte[encryptedBytesComposite.readableBytes()];
        encryptedBytesComposite.getBytes(encryptedBytesComposite.readerIndex(), arrayComposite);
        Assert.assertArrayEquals(array, arrayComposite);
        toEncrypt.release();
        toEncryptComposite.release();
        encryptedBytes.release();
        encryptedBytesComposite.release();
        ByteBuf toDecrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(array.length);
        toDecrypt.writeBytes(array);
        CompositeByteBuf toDecryptComposite = toDecrypt.alloc().compositeBuffer(size);
        size = array.length;
        start = 0;
        end = 0;
        for (int j = 0; j < 3; j++) {
            start = end;
            end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
            if (j == 2) {
                end = size;
            }
            ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
            c.writeBytes(array, start, end - start);
            toDecryptComposite.addComponent(true, c);
        }
        ByteBuf decryptedBytes = cryptoService.decrypt(toDecrypt, secretKeySpec);
        ByteBuf decryptedBytesComposite = cryptoService.decrypt(toDecryptComposite, secretKeySpec);
        Assert.assertEquals(decryptedBytes.readableBytes(), decryptedBytesComposite.readableBytes());
        Assert.assertEquals(toDecrypt.readableBytes(), 0);
        Assert.assertEquals(toDecryptComposite.readableBytes(), 0);
        array = new byte[decryptedBytes.readableBytes()];
        arrayComposite = new byte[decryptedBytesComposite.readableBytes()];
        decryptedBytes.getBytes(decryptedBytes.readerIndex(), array);
        decryptedBytesComposite.getBytes(decryptedBytesComposite.readerIndex(), arrayComposite);
        Assert.assertArrayEquals(array, arrayComposite);
        toDecrypt.release();
        toDecryptComposite.release();
        decryptedBytes.release();
        decryptedBytesComposite.release();
    }
}
Also used : CompositeByteBuf(io.netty.buffer.CompositeByteBuf) VerifiableProperties(com.github.ambry.config.VerifiableProperties) SecretKeySpec(javax.crypto.spec.SecretKeySpec) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) Test(org.junit.Test)

Aggregations

CompositeByteBuf (io.netty.buffer.CompositeByteBuf)86 ByteBuf (io.netty.buffer.ByteBuf)65 IOException (java.io.IOException)11 ArrayList (java.util.ArrayList)10 Test (org.junit.Test)8 ByteBuffer (java.nio.ByteBuffer)7 ChannelFuture (io.netty.channel.ChannelFuture)6 Channel (io.netty.channel.Channel)5 ChannelFutureListener (io.netty.channel.ChannelFutureListener)5 ChannelHandlerContext (io.netty.channel.ChannelHandlerContext)5 EmbeddedChannel (io.netty.channel.embedded.EmbeddedChannel)4 Test (org.junit.jupiter.api.Test)4 ChannelInboundHandlerAdapter (io.netty.channel.ChannelInboundHandlerAdapter)3 CodecException (io.netty.handler.codec.CodecException)3 SelfSignedCertificate (io.netty.handler.ssl.util.SelfSignedCertificate)3 InetSocketAddress (java.net.InetSocketAddress)3 ClosedChannelException (java.nio.channels.ClosedChannelException)3 List (java.util.List)3 ExecutionException (java.util.concurrent.ExecutionException)3 AtomicReference (java.util.concurrent.atomic.AtomicReference)3