use of org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf in project hbase by apache.
the class FanOutOneBlockAsyncDFSOutputHelper method requestWriteBlock.
private static void requestWriteBlock(Channel channel, Enum<?> storageType, OpWriteBlockProto.Builder writeBlockProtoBuilder) throws IOException {
OpWriteBlockProto proto = STORAGE_TYPE_SETTER.set(writeBlockProtoBuilder, storageType).build();
int protoLen = proto.getSerializedSize();
ByteBuf buffer = channel.alloc().buffer(3 + CodedOutputStream.computeRawVarint32Size(protoLen) + protoLen);
buffer.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION);
buffer.writeByte(Op.WRITE_BLOCK.code);
proto.writeDelimitedTo(new ByteBufOutputStream(buffer));
channel.writeAndFlush(buffer);
}
use of org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf in project hbase by apache.
the class FanOutOneBlockAsyncDFSOutput method flushBuffer.
private Promise<Void> flushBuffer(ByteBuf dataBuf, long nextPacketOffsetInBlock, boolean syncBlock) {
int dataLen = dataBuf.readableBytes();
int chunkLen = summer.getBytesPerChecksum();
int trailingPartialChunkLen = dataLen % chunkLen;
int numChecks = dataLen / chunkLen + (trailingPartialChunkLen != 0 ? 1 : 0);
int checksumLen = numChecks * summer.getChecksumSize();
ByteBuf checksumBuf = alloc.directBuffer(checksumLen);
summer.calculateChunkedSums(dataBuf.nioBuffer(), checksumBuf.nioBuffer(0, checksumLen));
checksumBuf.writerIndex(checksumLen);
PacketHeader header = new PacketHeader(4 + checksumLen + dataLen, nextPacketOffsetInBlock, nextPacketSeqno, false, dataLen, syncBlock);
int headerLen = header.getSerializedSize();
ByteBuf headerBuf = alloc.buffer(headerLen);
header.putInBuffer(headerBuf.nioBuffer(0, headerLen));
headerBuf.writerIndex(headerLen);
long ackedLength = nextPacketOffsetInBlock + dataLen;
Promise<Void> promise = eventLoop.<Void>newPromise().addListener(future -> {
if (future.isSuccess()) {
locatedBlock.getBlock().setNumBytes(ackedLength);
}
});
waitingAckQueue.addLast(new Callback(promise, ackedLength, datanodeList));
for (Channel ch : datanodeList) {
ch.write(headerBuf.duplicate().retain());
ch.write(checksumBuf.duplicate().retain());
ch.writeAndFlush(dataBuf.duplicate().retain());
}
checksumBuf.release();
headerBuf.release();
dataBuf.release();
nextPacketSeqno++;
return promise;
}
use of org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf in project pinot by linkedin.
the class NettySingleConnectionIntegrationTest method testSingleLargeRequestResponse.
@Test
public /**
* Test Single Large ( 2 MB) request response
* @throws Exception
*/
void testSingleLargeRequestResponse() throws Exception {
NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
final String response_prefix = "response_";
final String response = generatePayload(response_prefix, 1024 * 1024 * 2);
MyServer server = new MyServer(response);
Thread.sleep(1000);
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server.getServerInstance(), eventLoopGroup, new HashedWheelTimer(), metric);
try {
LOGGER.info("About to connect the client !!");
boolean connected = clientConn.connect();
LOGGER.info("Client connected !!");
Assert.assertTrue(connected, "connected");
Thread.sleep(1000);
String request_prefix = "request_";
String request = generatePayload(request_prefix, 1024 * 1024 * 2);
LOGGER.info("Sending the request !!");
ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
LOGGER.info("Request sent !!");
ByteBuf serverResp = serverRespFuture.getOne();
byte[] b2 = new byte[serverResp.readableBytes()];
serverResp.readBytes(b2);
String gotResponse = new String(b2);
Assert.assertTrue(gotResponse.equals(response), "Response Check at client");
Assert.assertTrue(server.getHandler().getRequest().equals(request), "Request Check at server");
} finally {
if (null != clientConn) {
clientConn.close();
}
server.shutdown();
}
}
use of org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf in project pinot by linkedin.
the class NettySingleConnectionIntegrationTest method testConcurrentRequestDispatchError.
@Test
public void testConcurrentRequestDispatchError() throws Exception {
NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
CountDownLatch latch = new CountDownLatch(1);
MyServer server = new MyServer();
Thread.sleep(1000);
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server.getServerInstance(), eventLoopGroup, new HashedWheelTimer(), metric);
LOGGER.info("About to connect the client !!");
boolean connected = clientConn.connect();
LOGGER.info("Client connected !!");
Assert.assertTrue(connected, "connected");
Thread.sleep(1000);
String request = "dummy request";
LOGGER.info("Sending the request !!");
ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
boolean gotException = false;
try {
clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
} catch (IllegalStateException ex) {
gotException = true;
// Second request should have failed.
LOGGER.info("got exception ", ex);
}
latch.countDown();
ByteBuf serverResp = serverRespFuture.getOne();
byte[] b2 = new byte[serverResp.readableBytes()];
serverResp.readBytes(b2);
String gotResponse = new String(b2);
Assert.assertEquals(gotResponse, server.getResponseStr(), "Response Check at client");
Assert.assertEquals(server.getHandler().getRequest(), request, "Request Check at server");
clientConn.close();
server.shutdown();
Assert.assertTrue(gotException, "GotException ");
}
use of org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf in project pinot by linkedin.
the class NettySingleConnectionIntegrationTest method test100LargeRequestResponses.
//@Test
//@Ignore
/**
* Send 100 large ( 2MB) sized request in sequence. Verify each request and response.
* @throws Exception
*/
//@Test
public void test100LargeRequestResponses() throws Exception {
NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
MyServer server = new MyServer(null);
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server.getServerInstance(), eventLoopGroup, new HashedWheelTimer(), metric);
LOGGER.info("About to connect the client !!");
boolean connected = clientConn.connect();
LOGGER.info("Client connected !!");
Assert.assertTrue(connected, "connected");
Thread.sleep(1000);
try {
for (int i = 0; i < 100; i++) {
String request_prefix = "request_";
String request = generatePayload(request_prefix, 1024 * 1024 * 20);
String response_prefix = "response_";
String response = generatePayload(response_prefix, 1024 * 1024 * 20);
server.getHandler().setResponse(response);
//LOG.info("Sending the request (" + request + ")");
ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
//LOG.info("Request sent !!");
ByteBuf serverResp = serverRespFuture.getOne();
byte[] b2 = new byte[serverResp.readableBytes()];
serverResp.readBytes(b2);
String gotResponse = new String(b2);
Assert.assertEquals(gotResponse, response, "Response Check at client");
Assert.assertEquals(server.getHandler().getRequest(), request, "Request Check at server");
}
} finally {
if (null != clientConn) {
clientConn.close();
}
server.shutdown();
}
}
Aggregations