Examples with SegmentIdSet com.linkedin.pinot.transport.common.SegmentIdSet used on opensource projects

Example 1 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherImpl method mergePartitionGroup.

private <T> void mergePartitionGroup(Map<T, SegmentIdSet> instanceToSegmentMap, T instances, SegmentId p) {
    SegmentIdSet pg2 = instanceToSegmentMap.get(instances);
    if (null != pg2) {
        pg2.addSegment(p);
    } else {
        SegmentIdSet pg1 = new SegmentIdSet();
        pg1.addSegment(p);
        instanceToSegmentMap.put(instances, pg1);
    }
}

Example 2 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherImpl method sendRequest.

/**
   *
   * Helper Function to send scatter-request. This method should be called after the servers are selected
   *
   * @param ctxt Scatter-Gather Request context with selected servers for each request.
   * @param scatterGatherStats scatter-gather statistics.
   * @param isOfflineTable whether the scatter-gather target is an OFFLINE table.
   * @param brokerMetrics broker metrics to track execution statistics.
   * @return a composite future representing the gather process.
   * @throws InterruptedException
   */
protected CompositeFuture<ServerInstance, ByteBuf> sendRequest(ScatterGatherRequestContext ctxt, ScatterGatherStats scatterGatherStats, Boolean isOfflineTable, BrokerMetrics brokerMetrics) throws InterruptedException {
    TimerContext t = MetricsHelper.startTimer();
    // Servers are expected to be selected at this stage
    Map<ServerInstance, SegmentIdSet> mp = ctxt.getSelectedServers();
    CountDownLatch requestDispatchLatch = new CountDownLatch(mp.size());
    // async checkout of connections and then dispatch of request
    List<SingleRequestHandler> handlers = new ArrayList<SingleRequestHandler>(mp.size());
    for (Entry<ServerInstance, SegmentIdSet> e : mp.entrySet()) {
        ServerInstance server = e.getKey();
        String serverName = server.toString();
        if (isOfflineTable != null) {
            if (isOfflineTable) {
                serverName += ScatterGatherStats.OFFLINE_TABLE_SUFFIX;
            } else {
                serverName += ScatterGatherStats.REALTIME_TABLE_SUFFIX;
            }
        }
        scatterGatherStats.initServer(serverName);
        SingleRequestHandler handler = new SingleRequestHandler(_connPool, server, ctxt.getRequest(), e.getValue(), ctxt.getTimeRemaining(), requestDispatchLatch, brokerMetrics);
        // Submit to thread-pool for checking-out and sending request
        _executorService.submit(handler);
        handlers.add(handler);
    }
    // Create the composite future for returning
    CompositeFuture<ServerInstance, ByteBuf> response = new CompositeFuture<ServerInstance, ByteBuf>("scatterRequest", GatherModeOnError.SHORTCIRCUIT_AND);
    // Wait for requests to be sent
    long timeRemaining = ctxt.getTimeRemaining();
    boolean sentSuccessfully = requestDispatchLatch.await(timeRemaining, TimeUnit.MILLISECONDS);
    if (sentSuccessfully) {
        List<KeyedFuture<ServerInstance, ByteBuf>> responseFutures = new ArrayList<KeyedFuture<ServerInstance, ByteBuf>>();
        for (SingleRequestHandler h : handlers) {
            responseFutures.add(h.getResponseFuture());
            String serverName = h.getServer().toString();
            if (isOfflineTable != null) {
                if (isOfflineTable) {
                    serverName += ScatterGatherStats.OFFLINE_TABLE_SUFFIX;
                } else {
                    serverName += ScatterGatherStats.REALTIME_TABLE_SUFFIX;
                }
            }
            scatterGatherStats.setSendStartTimeMillis(serverName, h.getConnStartTimeMillis());
            scatterGatherStats.setConnStartTimeMillis(serverName, h.getStartDelayMillis());
            scatterGatherStats.setSendCompletionTimeMillis(serverName, h.getSendCompletionTimeMillis());
        }
        response.start(responseFutures);
    } else {
        LOGGER.error("Request (" + ctxt.getRequest().getRequestId() + ") not sent completely within time (" + timeRemaining + " ms) !! Cancelling !!. NumSentFailed:" + requestDispatchLatch.getCount());
        response.start(null);
        // and so we cancel all of them here
        for (SingleRequestHandler h : handlers) {
            LOGGER.info("Request to {} was sent successfully:{}, cancelling.", h.getServer(), h.isSent());
            h.cancel();
        }
    }
    t.stop();
    _latency.update(t.getLatencyMs());
    return response;
}

Example 3 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherImpl method selectServicesPerPartition.

/**
   * For each segmentId in the instanceToSegmentMap, we select one (or more speculative) servers
   *
   * @param requestContext
   */
private void selectServicesPerPartition(ScatterGatherRequestContext requestContext) {
    Map<ServerInstance, SegmentIdSet> selectedServers = new HashMap<ServerInstance, SegmentIdSet>();
    ScatterGatherRequest request = requestContext.getRequest();
    Map<List<ServerInstance>, SegmentIdSet> instanceToSegmentMap = requestContext.getInvertedMap();
    ReplicaSelection selection = request.getReplicaSelection();
    for (Entry<List<ServerInstance>, SegmentIdSet> e : instanceToSegmentMap.entrySet()) {
        SegmentId firstPartition = null;
        for (SegmentId p : e.getValue().getSegments()) {
            /**
         * For selecting the server, we always use first segmentId in the group. This will provide
         * more chance for fanning out the query
         */
            if (null == firstPartition) {
                firstPartition = p;
            }
            ServerInstance s = selection.selectServer(firstPartition, e.getKey(), request.getHashKey());
            mergePartitionGroup(selectedServers, s, p);
        }
    }
    requestContext.setSelectedServers(selectedServers);
}

Example 4 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class RoutingTableTest method assertResourceRequest.

private void assertResourceRequest(HelixExternalViewBasedRouting routingTable, String resource, String expectedSegmentList, int expectedNumSegment) {
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resource, Collections.<String>emptyList());
    Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
    List<String> selectedSegments = new ArrayList<String>();
    for (ServerInstance serverInstance : serversMap.keySet()) {
        LOGGER.trace(serverInstance.toString());
        SegmentIdSet segmentIdSet = serversMap.get(serverInstance);
        LOGGER.trace(segmentIdSet.toString());
        selectedSegments.addAll(segmentIdSet.getSegmentsNameList());
    }
    String[] selectedSegmentArray = selectedSegments.toArray(new String[0]);
    Arrays.sort(selectedSegmentArray);
    Assert.assertEquals(selectedSegments.size(), expectedNumSegment);
    Assert.assertEquals(Arrays.toString(selectedSegmentArray), expectedSegmentList);
    LOGGER.trace("********************************");
}

Example 5 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerTimeout.

@Test
public void testMultipleServerTimeout() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7081;
    int serverPort2 = 7082;
    int serverPort3 = 7083;
    // Timeout server
    int serverPort4 = 7084;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 7000, false), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = new ThreadPoolExecutor(5, 5, 5, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, service, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    //Only 3 servers return value.
    Assert.assertEquals(v.size(), 3);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    //No  response from 4th server
    Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
    Map<ServerInstance, Throwable> errorMap = fut.getError();
    Assert.assertEquals(errorMap.size(), 1, "One error");
    Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
    Thread.sleep(3000);
    pool.getStats().refresh();
    Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
}