use of io.netty.buffer.CompositeByteBuf in project ambry by linkedin.
the class RetainingAsyncWritableChannel method writeInternal.
/**
* Internal method for writing to the channel that allows for a pluggable action to be taken to produce a
* retained {@link ByteBuf} that can be added to the composite buffer.
* @param retainedBufSupplier creates a retained {@link ByteBuf} that can be freely added to a
* {@link CompositeByteBuf}.
* @param callback called once the buffer has been added.
* @return a future that is completed once the buffer has been added.
*/
private Future<Long> writeInternal(Supplier<ByteBuf> retainedBufSupplier, Callback<Long> callback) {
FutureResult<Long> future = new FutureResult<>();
ByteBuf buf = null;
long bytesWritten = 0;
Exception exception = null;
try {
if (!isOpen()) {
throw new ClosedChannelException();
} else if (totalBytesWritten.get() > sizeLimitInBytes) {
throw new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
} else {
synchronized (bufferLock) {
if (compositeBuffer == null) {
throw new IllegalStateException("Cannot write more data; content already consumed or channel was closed");
}
buf = retainedBufSupplier.get();
bytesWritten = buf.readableBytes();
compositeBuffer.addComponent(true, buf);
}
if (totalBytesWritten.addAndGet(bytesWritten) > sizeLimitInBytes) {
exception = new RestServiceException("Request is larger than allowed size: " + sizeLimitInBytes, RestServiceErrorCode.RequestTooLarge);
}
}
} catch (Exception e) {
exception = e;
if (buf != null) {
buf.release();
}
} finally {
future.done(bytesWritten, exception);
if (callback != null) {
callback.onCompletion(bytesWritten, exception);
}
}
return future;
}
use of io.netty.buffer.CompositeByteBuf in project ambry by linkedin.
the class GetResponse method prepareBuffer.
/**
* A private method shared by {@link GetResponse#writeTo(WritableByteChannel)} and
* {@link GetResponse#writeTo(AsyncWritableChannel, Callback)}.
* This method allocate bufferToSend and write metadata to it if bufferToSend is null.
*/
@Override
protected void prepareBuffer() {
bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer((int) super.sizeInBytes() + (Partition_Response_Info_List_Size + partitionResponseInfoSize));
writeHeader();
if (partitionResponseInfoList != null) {
bufferToSend.writeInt(partitionResponseInfoList.size());
for (PartitionResponseInfo partitionResponseInfo : partitionResponseInfoList) {
partitionResponseInfo.writeTo(bufferToSend);
}
}
if (toSend != null) {
ByteBuf toSendContent = toSend.content();
if (toSendContent != null) {
// Since this composite blob will be a readonly blob, we don't really care about if it's allocated
// on a direct memory or not.
CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeDirectBuffer();
int maxNumComponent = 1 + toSendContent.nioBufferCount();
compositeByteBuf.addComponent(true, bufferToSend);
compositeByteBuf.addComponent(true, toSendContent);
bufferToSend = compositeByteBuf;
sendSizeInBufferToSend = toSendContent.readableBytes();
toSend = null;
}
}
}
use of io.netty.buffer.CompositeByteBuf in project ambry by linkedin.
the class PutRequest method prepareBuffer.
/**
* Construct the bufferToSend to serialize request metadata and other blob related information. The newly constructed
* bufferToSend will not include the blob content as it's carried by the {@code blob} field in this class.
*/
@Override
protected void prepareBuffer() {
// bufferToSend now is the header ByteBuf, it will store serialized header, without blob content and crc
bufferToSend = PooledByteBufAllocator.DEFAULT.ioBuffer(sizeExcludingBlobAndCrcSize());
writeHeader();
int crcStart = bufferToSend.writerIndex();
bufferToSend.writeBytes(blobId.toBytes());
BlobPropertiesSerDe.serializeBlobProperties(bufferToSend, properties);
bufferToSend.writeInt(usermetadata.capacity());
bufferToSend.writeBytes(usermetadata);
bufferToSend.writeShort((short) blobType.ordinal());
short keyLength = blobEncryptionKey == null ? 0 : (short) blobEncryptionKey.remaining();
bufferToSend.writeShort(keyLength);
if (keyLength > 0) {
bufferToSend.writeBytes(blobEncryptionKey);
}
bufferToSend.writeLong(blobSize);
// Now compute crc for the put request.
crc.update(bufferToSend.nioBuffer(crcStart, bufferToSend.writerIndex() - crcStart));
for (ByteBuffer bb : blob.nioBuffers()) {
crc.update(bb);
// change it back to 0 since we are going to write it to the channel later.
bb.position(0);
}
crcByteBuf = PooledByteBufAllocator.DEFAULT.ioBuffer(CRC_SIZE_IN_BYTES);
crcByteBuf.writeLong(crc.getValue());
// Now construct the real bufferToSend, which should be a composite ByteBuf.
CompositeByteBuf compositeByteBuf = bufferToSend.alloc().compositeHeapBuffer(2 + blob.nioBufferCount());
compositeByteBuf.addComponent(true, bufferToSend);
if (blob instanceof CompositeByteBuf) {
Iterator<ByteBuf> iter = ((CompositeByteBuf) blob).iterator();
while (iter.hasNext()) {
compositeByteBuf.addComponent(true, iter.next());
}
} else {
compositeByteBuf.addComponent(true, blob);
}
compositeByteBuf.addComponent(true, crcByteBuf);
blob = null;
crcByteBuf = null;
bufferToSend = compositeByteBuf;
}
use of io.netty.buffer.CompositeByteBuf in project ambry by linkedin.
the class CopyForcingByteBuf method generateCompositeContent.
/**
* Generates random content and fills it up in {@code httpContents} with a backing {@link CompositeByteBuf}.
* @param httpContents the {@link List<HttpContent>} that will contain all the content.
* @return the whole content as a {@link ByteBuffer} - serves as a source of truth.
*/
private ByteBuffer generateCompositeContent(List<HttpContent> httpContents) {
int individualPartSize = GENERATED_CONTENT_SIZE / GENERATED_CONTENT_PART_COUNT;
byte[] contentBytes = TestUtils.getRandomBytes(GENERATED_CONTENT_SIZE);
ArrayList<ByteBuf> byteBufs = new ArrayList<>(GENERATED_CONTENT_PART_COUNT);
for (int addedContentCount = 0; addedContentCount < GENERATED_CONTENT_PART_COUNT; addedContentCount++) {
byteBufs.add(Unpooled.wrappedBuffer(contentBytes, addedContentCount * individualPartSize, individualPartSize));
}
httpContents.add(new DefaultLastHttpContent(new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 20, byteBufs)));
return ByteBuffer.wrap(contentBytes);
}
use of io.netty.buffer.CompositeByteBuf in project ambry by linkedin.
the class GCMCryptoServiceTest method testEncryptDecryptNettyCompositeByteBuf.
@Test
public void testEncryptDecryptNettyCompositeByteBuf() throws Exception {
// testEncryptDecryptNettyByte already tests the correctness of the encrypt decrypt methods with non-composite Netty
// ByteBuf, in this test case, we can make the assumption that these two functions always provide correct answers.
String key = TestUtils.getRandomKey(DEFAULT_KEY_SIZE_IN_CHARS);
Properties props = getKMSProperties(key, DEFAULT_KEY_SIZE_IN_CHARS);
VerifiableProperties verifiableProperties = new VerifiableProperties((props));
SecretKeySpec secretKeySpec = new SecretKeySpec(Hex.decode(key), "AES");
GCMCryptoService cryptoService = (GCMCryptoService) (new GCMCryptoServiceFactory(verifiableProperties, REGISTRY).getCryptoService());
byte[] fixedIv = new byte[12];
for (int i = 0; i < 5; i++) {
int size = TestUtils.RANDOM.nextInt(MAX_DATA_SIZE - MIN_DATA_SIZE) + MIN_DATA_SIZE;
byte[] randomData = new byte[size];
TestUtils.RANDOM.nextBytes(randomData);
ByteBuf toEncrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(randomData.length);
toEncrypt.writeBytes(randomData);
CompositeByteBuf toEncryptComposite = toEncrypt.alloc().compositeBuffer(size);
int start = 0;
int end = 0;
for (int j = 0; j < 3; j++) {
start = end;
end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
if (j == 2) {
end = size;
}
ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
c.writeBytes(randomData, start, end - start);
toEncryptComposite.addComponent(true, c);
}
ByteBuf encryptedBytes = cryptoService.encrypt(toEncrypt, secretKeySpec, fixedIv);
ByteBuf encryptedBytesComposite = cryptoService.encrypt(toEncryptComposite, secretKeySpec, fixedIv);
Assert.assertEquals(encryptedBytes.readableBytes(), encryptedBytesComposite.readableBytes());
Assert.assertEquals(toEncrypt.readableBytes(), 0);
Assert.assertEquals(toEncryptComposite.readableBytes(), 0);
byte[] array = new byte[encryptedBytes.readableBytes()];
encryptedBytes.getBytes(encryptedBytes.readerIndex(), array);
byte[] arrayComposite = new byte[encryptedBytesComposite.readableBytes()];
encryptedBytesComposite.getBytes(encryptedBytesComposite.readerIndex(), arrayComposite);
Assert.assertArrayEquals(array, arrayComposite);
toEncrypt.release();
toEncryptComposite.release();
encryptedBytes.release();
encryptedBytesComposite.release();
ByteBuf toDecrypt = PooledByteBufAllocator.DEFAULT.ioBuffer(array.length);
toDecrypt.writeBytes(array);
CompositeByteBuf toDecryptComposite = toDecrypt.alloc().compositeBuffer(size);
size = array.length;
start = 0;
end = 0;
for (int j = 0; j < 3; j++) {
start = end;
end = TestUtils.RANDOM.nextInt(size / 2 - 1) + end;
if (j == 2) {
end = size;
}
ByteBuf c = PooledByteBufAllocator.DEFAULT.ioBuffer(end - start);
c.writeBytes(array, start, end - start);
toDecryptComposite.addComponent(true, c);
}
ByteBuf decryptedBytes = cryptoService.decrypt(toDecrypt, secretKeySpec);
ByteBuf decryptedBytesComposite = cryptoService.decrypt(toDecryptComposite, secretKeySpec);
Assert.assertEquals(decryptedBytes.readableBytes(), decryptedBytesComposite.readableBytes());
Assert.assertEquals(toDecrypt.readableBytes(), 0);
Assert.assertEquals(toDecryptComposite.readableBytes(), 0);
array = new byte[decryptedBytes.readableBytes()];
arrayComposite = new byte[decryptedBytesComposite.readableBytes()];
decryptedBytes.getBytes(decryptedBytes.readerIndex(), array);
decryptedBytesComposite.getBytes(decryptedBytesComposite.readerIndex(), arrayComposite);
Assert.assertArrayEquals(array, arrayComposite);
toDecrypt.release();
toDecryptComposite.release();
decryptedBytes.release();
decryptedBytesComposite.release();
}
}
Aggregations