Examples with ServerInstance com.linkedin.pinot.common.response.ServerInstance used on opensource projects

Example 36 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ApproximateQueryTestUtil method runQuery.

public static Object runQuery(QueryExecutor queryExecutor, List<String> segments, AvroQueryGenerator.TestAggreationQuery query, ServerMetrics metrics) {
    LOGGER.info("\nRunning: " + query.getPql());
    final BrokerRequest brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(query.getPql());
    InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (String segment : segments) {
        instanceRequest.getSearchSegments().add(segment);
    }
    QueryRequest queryRequest = new QueryRequest(instanceRequest, metrics);
    final DataTable instanceResponse = queryExecutor.processQuery(queryRequest, queryRunners);
    final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    final BrokerResponseNative brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
    AggregationResult result = brokerResponse.getAggregationResults().get(0);
    Assert.assertNotNull(result);
    if (result.getValue() != null) {
        LOGGER.info("Aggregation Result is " + result.getValue().toString());
    } else if (result.getGroupByResult() != null) {
        LOGGER.info("GroupBy Result is " + result.getGroupByResult().toString());
    } else {
        throw new RuntimeException("Aggregation and GroupBy Results both null.");
    }
    // compute value
    Object val;
    if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
        val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getValue().toString());
    } else {
        val = brokerResponse.getAggregationResults().get(0).getGroupByResult();
    }
    return val;
}

Example 37 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ScatterGatherTest method testSingleServer.

@Test
public void testSingleServer() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort = 7071;
    NettyTCPServer server1 = new NettyTCPServer(serverPort, new TestRequestHandlerFactory(0, 1), null);
    Thread t1 = new Thread(server1);
    t1.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    SegmentIdSet pg = new SegmentIdSet();
    pg.addSegment(new SegmentId("0"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort);
    List<ServerInstance> instances = new ArrayList<ServerInstance>();
    instances.add(serverInstance1);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg);
    String request = "request_0";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg, request);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    Assert.assertEquals(v.size(), 1);
    server1.shutdownGracefully();
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
}

Example 38 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerError.

@Test
public void testMultipleServerError() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7091;
    int serverPort2 = 7092;
    int serverPort3 = 7093;
    // error server
    int serverPort4 = 7094;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 1000, true), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    final BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    //Only 3 servers return value.
    Assert.assertEquals(v.size(), 3);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    //No  response from 4th server
    Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
    Map<ServerInstance, Throwable> errorMap = fut.getError();
    Assert.assertEquals(errorMap.size(), 1, "One error");
    Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
    //    System.out.println("Error is :" + errorMap.get(serverInstance4));
    Thread.sleep(3000);
    //    System.out.println("Pool Stats :" + pool.getStats());
    pool.getStats().refresh();
    Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
}

Example 39 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class NettyCloseChannelTest method testCloseClientChannel.

@Test
public /**
   * Client sends a request. Before Server generates the response, the client closes the channel (scenario:as the client is shutting down)
   */
void testCloseClientChannel() throws Exception {
    NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
    Timer timer = new HashedWheelTimer();
    String response = "dummy response";
    int port = 9089;
    CountDownLatch latch = new CountDownLatch(1);
    MyRequestHandler handler = new MyRequestHandler(response, latch);
    MyRequestHandlerFactory handlerFactory = new MyRequestHandlerFactory(handler);
    NettyTCPServer serverConn = new NettyTCPServer(port, handlerFactory, null);
    Thread serverThread = new Thread(serverConn, "ServerMain");
    serverThread.start();
    Thread.sleep(1000);
    ServerInstance server = new ServerInstance("localhost", port);
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server, eventLoopGroup, timer, metric);
    LOGGER.debug("About to connect the client !!");
    boolean connected = clientConn.connect();
    LOGGER.debug("Client connected !!");
    Assert.assertTrue(connected, "connected");
    Thread.sleep(1000);
    String request = "dummy request";
    LOGGER.debug("Sending the request !!");
    ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
    //Close the client
    clientConn.close();
    latch.countDown();
    ByteBuf serverResp = serverRespFuture.getOne();
    clientConn.close();
    serverConn.shutdownGracefully();
    Assert.assertNull(serverResp);
    Assert.assertFalse(serverRespFuture.isCancelled(), "Is Cancelled");
    Assert.assertTrue(serverRespFuture.getError() != null, "Got Exception");
    serverConn.shutdownGracefully();
    LOGGER.trace("metrics: {}", metric);
}

Example 40 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class NettyCloseChannelTest method testCloseServerChannel.

@Test
public /**
   * Client sends a request. Server closes the channel ( scenario: as it is shutting down)
   */
void testCloseServerChannel() throws Exception {
    NettyClientMetrics metric = new NettyClientMetrics(null, "abc");
    Timer timer = new HashedWheelTimer();
    int port = 9089;
    MyRequestHandler handler = new MyRequestHandler("empty", null);
    MyRequestHandlerFactory handlerFactory = new MyRequestHandlerFactory(handler);
    NettyTCPServer serverConn = new NettyCloseTCPServer(port, handlerFactory);
    Thread serverThread = new Thread(serverConn, "ServerMain");
    serverThread.start();
    Thread.sleep(1000);
    ServerInstance server = new ServerInstance("localhost", port);
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyTCPClientConnection clientConn = new NettyTCPClientConnection(server, eventLoopGroup, timer, metric);
    LOGGER.info("About to connect the client !!");
    boolean connected = clientConn.connect();
    LOGGER.info("Client connected !!");
    Assert.assertTrue(connected, "connected");
    Thread.sleep(1000);
    String request = "dummy request";
    LOGGER.info("Sending the request !!");
    ResponseFuture serverRespFuture = clientConn.sendRequest(Unpooled.wrappedBuffer(request.getBytes()), 1L, 5000L);
    ByteBuf serverResp = serverRespFuture.getOne();
    clientConn.close();
    serverConn.shutdownGracefully();
    Assert.assertNull(serverResp);
    Assert.assertFalse(serverRespFuture.isCancelled(), "Is Cancelled");
    Assert.assertTrue(serverRespFuture.getError() != null, "Got Exception");
    LOGGER.trace("metrics: {}", metric);
}