Examples with SegmentIdSet com.linkedin.pinot.transport.common.SegmentIdSet used on opensource projects

Example 6 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerHappy.

@Test
public void testMultipleServerHappy() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7071;
    int serverPort2 = 7072;
    int serverPort3 = 7073;
    int serverPort4 = 7074;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    Assert.assertEquals(v.size(), 4);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    b = v.get(serverInstance4);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_3_0");
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
}

Example 7 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class RoutingTableTest method assertResourceRequest.

private void assertResourceRequest(HelixExternalViewBasedRouting routingTable, String resource, String[] expectedSegmentLists, int expectedNumSegment) {
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resource, Collections.<String>emptyList());
    Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
    List<String> selectedSegments = new ArrayList<String>();
    for (ServerInstance serverInstance : serversMap.keySet()) {
        LOGGER.trace(serverInstance.toString());
        SegmentIdSet segmentIdSet = serversMap.get(serverInstance);
        LOGGER.trace(segmentIdSet.toString());
        selectedSegments.addAll(segmentIdSet.getSegmentsNameList());
    }
    String[] selectedSegmentArray = selectedSegments.toArray(new String[0]);
    Arrays.sort(selectedSegmentArray);
    Assert.assertEquals(selectedSegments.size(), expectedNumSegment);
    boolean matchedExpectedLists = false;
    for (String expectedSegmentList : expectedSegmentLists) {
        if (expectedSegmentList.equals(Arrays.toString(selectedSegmentArray))) {
            matchedExpectedLists = true;
        }
    }
    Assert.assertTrue(matchedExpectedLists);
    LOGGER.trace("********************************");
}

Example 8 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class RoutingTableTest method testCombinedKafkaRouting.

// Test that we can switch between llc and hlc routing depending on what the selector tells us.
@Test
public void testCombinedKafkaRouting() throws Exception {
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    final long now = System.currentTimeMillis();
    final String tableName = "table";
    final String resourceName = tableName + "_REALTIME";
    final String group1 = resourceName + "_" + Long.toString(now) + "_0";
    final String group2 = resourceName + "_" + Long.toString(now) + "_1";
    final String online = "ONLINE";
    final String consuming = "CONSUMING";
    final int partitionId = 1;
    final String partitionRange = "JUNK";
    final int segId1 = 1;
    final int segId2 = 2;
    final int port1 = 1;
    final int port2 = 2;
    final String host = "host";
    final ServerInstance serverInstance1 = new ServerInstance(host, port1);
    final ServerInstance serverInstance2 = new ServerInstance(host, port2);
    final String helixInstance1 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance1;
    final String helixInstance2 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance2;
    final HLCSegmentName s1HlcSegment1 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s1HlcSegment2 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId2));
    final HLCSegmentName s2HlcSegment1 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s2HlcSegment2 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId2));
    final LLCSegmentName llcSegment1 = new LLCSegmentName(tableName, partitionId, segId1, now);
    final LLCSegmentName llcSegment2 = new LLCSegmentName(tableName, partitionId, segId2, now);
    final List<InstanceConfig> instanceConfigs = new ArrayList<>(2);
    instanceConfigs.add(new InstanceConfig(helixInstance1));
    instanceConfigs.add(new InstanceConfig(helixInstance2));
    ExternalView ev = new ExternalView(resourceName);
    ev.setState(s1HlcSegment1.getSegmentName(), helixInstance1, online);
    ev.setState(s1HlcSegment2.getSegmentName(), helixInstance1, online);
    ev.setState(llcSegment1.getSegmentName(), helixInstance2, online);
    ev.setState(llcSegment2.getSegmentName(), helixInstance2, consuming);
    routingTable.markDataResourceOnline(resourceName, ev, instanceConfigs);
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resourceName, Collections.<String>emptyList());
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
    }
    // Now change the percent value in the routing table selector to be 100, and we should get only LLC segments.
    Configuration configuration = new PropertiesConfiguration();
    configuration.addProperty("class", PercentageBasedRoutingTableSelector.class.getName());
    configuration.addProperty("table." + resourceName, new Integer(100));
    RoutingTableSelector selector = RoutingTableSelectorFactory.getRoutingTableSelector(configuration, null);
    selector.init(configuration, null);
    Field selectorField = HelixExternalViewBasedRouting.class.getDeclaredField("_routingTableSelector");
    selectorField.setAccessible(true);
    selectorField.set(routingTable, selector);
    // And we should find only LLC segments.
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
    }
    // Now change it to 50, and we should find both (at least 10 times each).
    configuration = new PropertiesConfiguration();
    configuration.addProperty("table." + resourceName, new Integer(50));
    selector = new PercentageBasedRoutingTableSelector();
    selector.init(configuration, null);
    selectorField.set(routingTable, selector);
    int hlc = 0;
    int llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    // If we do the above iteration 100 times, we should get at least 10 of each type of routing.
    // If this test fails
    Assert.assertTrue(hlc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    Assert.assertTrue(llc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    // Check that force HLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_HLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 100);
    Assert.assertEquals(llc, 0);
    // Check that force LLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_LLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 0);
    Assert.assertEquals(llc, 100);
}

Example 9 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherImpl method buildInvertedMap.

/**
   * Merge segment-sets which have the same set of servers. If 2 segmentIds have overlapping
   * set of servers, they are not merged. If there is predefined-selection for a segmentId,
   * a separate entry is added for those in the inverted map.
   * @param requestContext Scatter gather request
   */
protected void buildInvertedMap(ScatterGatherRequestContext requestContext) {
    ScatterGatherRequest request = requestContext.getRequest();
    Map<ServerInstance, SegmentIdSet> segmentIdToInstanceMap = request.getSegmentsServicesMap();
    Map<List<ServerInstance>, SegmentIdSet> instanceToSegmentMap = new HashMap<List<ServerInstance>, SegmentIdSet>();
    for (ServerInstance serverInstance : segmentIdToInstanceMap.keySet()) {
        instanceToSegmentMap.put(Arrays.asList(serverInstance), segmentIdToInstanceMap.get(serverInstance));
    }
    requestContext.setInvertedMap(instanceToSegmentMap);
}

Example 10 with SegmentIdSet

use of com.linkedin.pinot.transport.common.SegmentIdSet in project pinot by linkedin.

the class ScatterGatherImpl method selectServicesPerPartitionGroup.

/**
   * For each segment-set in the instanceToSegmentMap, we select one (or more speculative) servers
   *
   * @param requestContext
   */
private void selectServicesPerPartitionGroup(ScatterGatherRequestContext requestContext) {
    Map<ServerInstance, SegmentIdSet> selectedServers = new HashMap<ServerInstance, SegmentIdSet>();
    ScatterGatherRequest request = requestContext.getRequest();
    Map<List<ServerInstance>, SegmentIdSet> instanceToSegmentMap = requestContext.getInvertedMap();
    //int numDuplicateRequests = request.getNumSpeculativeRequests();
    ReplicaSelection selection = request.getReplicaSelection();
    for (Entry<List<ServerInstance>, SegmentIdSet> e : instanceToSegmentMap.entrySet()) {
        ServerInstance s = selection.selectServer(e.getValue().getOneSegment(), e.getKey(), request.getHashKey());
        mergePartitionGroup(selectedServers, s, e.getValue());
    /**
       * TODO:
       * We can easily add speculative execution here. The below code will pick a distinct server
       * for the segmentId, This entry needs to be maintained in a separate container in ScatterGatherRequestContext
       * Then in sndRequest, we need to construct SelectingFuture for the pairs of Future corresponding to original
       * and speculative(duplicate) request.
       *
       int numServers = e.getKey().size();

      // Pick Unique servers for speculative request
      int numIterations = Math.min(numServers - 1, numDuplicateRequests);
      for (int i = 0, c = 0; i < numIterations; i++, c++)
      {
        ServerInstance s1 = e.getKey().get(c);
        if ( s.equals(s1))
        {
          c++;
          s1 = e.getKey().get(c);
        }
        mergePartitionGroup(selectedServers, s1, e.getValue());
        //TODO: speculative servers need to be maintained in a separate entry in ScatterGatherRequestContext
      }
       **/
    }
    requestContext.setSelectedServers(selectedServers);
}