use of com.linkedin.pinot.transport.metrics.NettyClientMetrics in project pinot by linkedin.
the class ScatterGatherTest method testMultipleServerHappy.
@Test
public void testMultipleServerHappy() throws Exception {
MetricsRegistry registry = new MetricsRegistry();
// Server start
int serverPort1 = 7071;
int serverPort2 = 7072;
int serverPort3 = 7073;
int serverPort4 = 7074;
NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1), null);
Thread t1 = new Thread(server1);
Thread t2 = new Thread(server2);
Thread t3 = new Thread(server3);
Thread t4 = new Thread(server4);
t1.start();
t2.start();
t3.start();
t4.start();
//Client setup
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
rm.setPool(pool);
SegmentIdSet pg1 = new SegmentIdSet();
pg1.addSegment(new SegmentId("0"));
SegmentIdSet pg2 = new SegmentIdSet();
pg2.addSegment(new SegmentId("1"));
SegmentIdSet pg3 = new SegmentIdSet();
pg3.addSegment(new SegmentId("2"));
SegmentIdSet pg4 = new SegmentIdSet();
pg4.addSegment(new SegmentId("3"));
ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
pgMap.put(serverInstance1, pg1);
pgMap.put(serverInstance2, pg2);
pgMap.put(serverInstance3, pg3);
pgMap.put(serverInstance4, pg4);
String request1 = "request_0";
String request2 = "request_1";
String request3 = "request_2";
String request4 = "request_3";
Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
pgMapStr.put(pg1, request1);
pgMapStr.put(pg2, request2);
pgMapStr.put(pg3, request3);
pgMapStr.put(pg4, request4);
ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
Map<ServerInstance, ByteBuf> v = fut.get();
Assert.assertEquals(v.size(), 4);
ByteBuf b = v.get(serverInstance1);
byte[] b2 = new byte[b.readableBytes()];
b.readBytes(b2);
String response = new String(b2);
Assert.assertEquals(response, "response_0_0");
b = v.get(serverInstance2);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_1_0");
b = v.get(serverInstance3);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_2_0");
b = v.get(serverInstance4);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_3_0");
server1.shutdownGracefully();
server2.shutdownGracefully();
server3.shutdownGracefully();
server4.shutdownGracefully();
pool.shutdown();
service.shutdown();
eventLoopGroup.shutdownGracefully();
}
use of com.linkedin.pinot.transport.metrics.NettyClientMetrics in project pinot by linkedin.
the class ScatterGatherTest method testSingleServer.
@Test
public void testSingleServer() throws Exception {
MetricsRegistry registry = new MetricsRegistry();
// Server start
int serverPort = 7071;
NettyTCPServer server1 = new NettyTCPServer(serverPort, new TestRequestHandlerFactory(0, 1), null);
Thread t1 = new Thread(server1);
t1.start();
//Client setup
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
rm.setPool(pool);
ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
SegmentIdSet pg = new SegmentIdSet();
pg.addSegment(new SegmentId("0"));
ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort);
List<ServerInstance> instances = new ArrayList<ServerInstance>();
instances.add(serverInstance1);
Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
pgMap.put(serverInstance1, pg);
String request = "request_0";
Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
pgMapStr.put(pg, request);
ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
Map<ServerInstance, ByteBuf> v = fut.get();
ByteBuf b = v.get(serverInstance1);
byte[] b2 = new byte[b.readableBytes()];
b.readBytes(b2);
String response = new String(b2);
Assert.assertEquals(response, "response_0_0");
Assert.assertEquals(v.size(), 1);
server1.shutdownGracefully();
pool.shutdown();
service.shutdown();
eventLoopGroup.shutdownGracefully();
}
use of com.linkedin.pinot.transport.metrics.NettyClientMetrics in project pinot by linkedin.
the class ScatterGatherTest method testMultipleServerError.
@Test
public void testMultipleServerError() throws Exception {
MetricsRegistry registry = new MetricsRegistry();
// Server start
int serverPort1 = 7091;
int serverPort2 = 7092;
int serverPort3 = 7093;
// error server
int serverPort4 = 7094;
NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 1000, true), null);
Thread t1 = new Thread(server1);
Thread t2 = new Thread(server2);
Thread t3 = new Thread(server3);
Thread t4 = new Thread(server4);
t1.start();
t2.start();
t3.start();
t4.start();
//Client setup
ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
rm.setPool(pool);
SegmentIdSet pg1 = new SegmentIdSet();
pg1.addSegment(new SegmentId("0"));
SegmentIdSet pg2 = new SegmentIdSet();
pg2.addSegment(new SegmentId("1"));
SegmentIdSet pg3 = new SegmentIdSet();
pg3.addSegment(new SegmentId("2"));
SegmentIdSet pg4 = new SegmentIdSet();
pg4.addSegment(new SegmentId("3"));
ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
pgMap.put(serverInstance1, pg1);
pgMap.put(serverInstance2, pg2);
pgMap.put(serverInstance3, pg3);
pgMap.put(serverInstance4, pg4);
String request1 = "request_0";
String request2 = "request_1";
String request3 = "request_2";
String request4 = "request_3";
Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
pgMapStr.put(pg1, request1);
pgMapStr.put(pg2, request2);
pgMapStr.put(pg3, request3);
pgMapStr.put(pg4, request4);
ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
final BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
Map<ServerInstance, ByteBuf> v = fut.get();
//Only 3 servers return value.
Assert.assertEquals(v.size(), 3);
ByteBuf b = v.get(serverInstance1);
byte[] b2 = new byte[b.readableBytes()];
b.readBytes(b2);
String response = new String(b2);
Assert.assertEquals(response, "response_0_0");
b = v.get(serverInstance2);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_1_0");
b = v.get(serverInstance3);
b2 = new byte[b.readableBytes()];
b.readBytes(b2);
response = new String(b2);
Assert.assertEquals(response, "response_2_0");
//No response from 4th server
Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
Map<ServerInstance, Throwable> errorMap = fut.getError();
Assert.assertEquals(errorMap.size(), 1, "One error");
Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
// System.out.println("Error is :" + errorMap.get(serverInstance4));
Thread.sleep(3000);
// System.out.println("Pool Stats :" + pool.getStats());
pool.getStats().refresh();
Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
pool.shutdown();
service.shutdown();
eventLoopGroup.shutdownGracefully();
server1.shutdownGracefully();
server2.shutdownGracefully();
server3.shutdownGracefully();
server4.shutdownGracefully();
}
use of com.linkedin.pinot.transport.metrics.NettyClientMetrics in project pinot by linkedin.
the class NettyCloseChannelTest method testCloseClientChannel.
@Test
public /**
* Client sends a request. Before Server generates the response, the client closes the channel (scenario:as the client is shutting down)
*/
void testCloseClientChannel() throws Exception {
NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
Timer timer = new HashedWheelTimer();
String response = "dummy response";
int port = 9089;
CountDownLatch latch = new CountDownLatch(1);
MyRequestHandler handler = new MyRequestHandler(response, latch);
MyRequestHandlerFactory handlerFactory = new MyRequestHandlerFactory(handler);
NettyTCPServer serverConn = new NettyTCPServer(port, handlerFactory, null);
Thread serverThread = new Thread(serverConn, "ServerMain");
serverThread.start();
Thread.sleep(1000);
ServerInstance server = new ServerInstance("localhost", port);
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server, eventLoopGroup, timer, metric);
LOGGER.debug("About to connect the client !!");
boolean connected = clientConn.connect();
LOGGER.debug("Client connected !!");
Assert.assertTrue(connected, "connected");
Thread.sleep(1000);
String request = "dummy request";
LOGGER.debug("Sending the request !!");
ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
//Close the client
clientConn.close();
latch.countDown();
ByteBuf serverResp = serverRespFuture.getOne();
clientConn.close();
serverConn.shutdownGracefully();
Assert.assertNull(serverResp);
Assert.assertFalse(serverRespFuture.isCancelled(), "Is Cancelled");
Assert.assertTrue(serverRespFuture.getError() != null, "Got Exception");
serverConn.shutdownGracefully();
LOGGER.trace("metrics: {}", metric);
}
use of com.linkedin.pinot.transport.metrics.NettyClientMetrics in project pinot by linkedin.
the class NettyCloseChannelTest method testCloseServerChannel.
@Test
public /**
* Client sends a request. Server closes the channel ( scenario: as it is shutting down)
*/
void testCloseServerChannel() throws Exception {
NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
Timer timer = new HashedWheelTimer();
int port = 9089;
MyRequestHandler handler = new MyRequestHandler("empty", null);
MyRequestHandlerFactory handlerFactory = new MyRequestHandlerFactory(handler);
NettyTCPServer serverConn = new NettyCloseTCPServer(port, handlerFactory);
Thread serverThread = new Thread(serverConn, "ServerMain");
serverThread.start();
Thread.sleep(1000);
ServerInstance server = new ServerInstance("localhost", port);
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server, eventLoopGroup, timer, metric);
LOGGER.info("About to connect the client !!");
boolean connected = clientConn.connect();
LOGGER.info("Client connected !!");
Assert.assertTrue(connected, "connected");
Thread.sleep(1000);
String request = "dummy request";
LOGGER.info("Sending the request !!");
ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
ByteBuf serverResp = serverRespFuture.getOne();
clientConn.close();
serverConn.shutdownGracefully();
Assert.assertNull(serverResp);
Assert.assertFalse(serverRespFuture.isCancelled(), "Is Cancelled");
Assert.assertTrue(serverRespFuture.getError() != null, "Got Exception");
LOGGER.trace("metrics: {}", metric);
}
Aggregations