Examples with ServerInstance com.linkedin.pinot.common.response.ServerInstance used on opensource projects

Example 26 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class BrokerReduceServiceTest method testMultiAggregationQuery.

@Test
public void testMultiAggregationQuery() {
    BrokerRequest brokerRequest = getMultiAggregationQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        instanceResponseMap.put(new ServerInstance("localhost:0000"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:1111"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:2222"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:3333"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:4444"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:5555"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:6666"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:7777"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:8888"), _queryExecutor.processQuery(queryRequest, queryRunners));
        instanceResponseMap.put(new ServerInstance("localhost:9999"), _queryExecutor.processQuery(queryRequest, queryRunners));
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "count_star", 4000020.0);
        aggregationResult = brokerResponse.getAggregationResults().get(1);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "sum_met", 400002000000.0);
        aggregationResult = brokerResponse.getAggregationResults().get(2);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "max_met", 200000.0);
        aggregationResult = brokerResponse.getAggregationResults().get(3);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "min_met", 0.0);
        aggregationResult = brokerResponse.getAggregationResults().get(4);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "avg_met", 100000.0);
        aggregationResult = brokerResponse.getAggregationResults().get(5);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "distinctCount_dim0", 10.0);
        aggregationResult = brokerResponse.getAggregationResults().get(6);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "distinctCount_dim1", 100.0);
        LOGGER.info("Time Used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
        LOGGER.info("Num Docs Scanned is " + brokerResponse.getNumDocsScanned());
        LOGGER.info("Total Docs for BrokerResponse is " + brokerResponse.getTotalDocs());
    //      System.out.println(brokerResponse.toJson());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}

Example 27 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class RoutingTableTest method assertResourceRequest.

private void assertResourceRequest(HelixExternalViewBasedRouting routingTable, String resource, String[] expectedSegmentLists, int expectedNumSegment) {
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resource, Collections.<String>emptyList());
    Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
    List<String> selectedSegments = new ArrayList<String>();
    for (ServerInstance serverInstance : serversMap.keySet()) {
        LOGGER.trace(serverInstance.toString());
        SegmentIdSet segmentIdSet = serversMap.get(serverInstance);
        LOGGER.trace(segmentIdSet.toString());
        selectedSegments.addAll(segmentIdSet.getSegmentsNameList());
    }
    String[] selectedSegmentArray = selectedSegments.toArray(new String[0]);
    Arrays.sort(selectedSegmentArray);
    Assert.assertEquals(selectedSegments.size(), expectedNumSegment);
    boolean matchedExpectedLists = false;
    for (String expectedSegmentList : expectedSegmentLists) {
        if (expectedSegmentList.equals(Arrays.toString(selectedSegmentArray))) {
            matchedExpectedLists = true;
        }
    }
    Assert.assertTrue(matchedExpectedLists);
    LOGGER.trace("********************************");
}

Example 28 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class RoutingTableTest method testCombinedKafkaRouting.

// Test that we can switch between llc and hlc routing depending on what the selector tells us.
@Test
public void testCombinedKafkaRouting() throws Exception {
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    final long now = System.currentTimeMillis();
    final String tableName = "table";
    final String resourceName = tableName + "_REALTIME";
    final String group1 = resourceName + "_" + Long.toString(now) + "_0";
    final String group2 = resourceName + "_" + Long.toString(now) + "_1";
    final String online = "ONLINE";
    final String consuming = "CONSUMING";
    final int partitionId = 1;
    final String partitionRange = "JUNK";
    final int segId1 = 1;
    final int segId2 = 2;
    final int port1 = 1;
    final int port2 = 2;
    final String host = "host";
    final ServerInstance serverInstance1 = new ServerInstance(host, port1);
    final ServerInstance serverInstance2 = new ServerInstance(host, port2);
    final String helixInstance1 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance1;
    final String helixInstance2 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance2;
    final HLCSegmentName s1HlcSegment1 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s1HlcSegment2 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId2));
    final HLCSegmentName s2HlcSegment1 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s2HlcSegment2 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId2));
    final LLCSegmentName llcSegment1 = new LLCSegmentName(tableName, partitionId, segId1, now);
    final LLCSegmentName llcSegment2 = new LLCSegmentName(tableName, partitionId, segId2, now);
    final List<InstanceConfig> instanceConfigs = new ArrayList<>(2);
    instanceConfigs.add(new InstanceConfig(helixInstance1));
    instanceConfigs.add(new InstanceConfig(helixInstance2));
    ExternalView ev = new ExternalView(resourceName);
    ev.setState(s1HlcSegment1.getSegmentName(), helixInstance1, online);
    ev.setState(s1HlcSegment2.getSegmentName(), helixInstance1, online);
    ev.setState(llcSegment1.getSegmentName(), helixInstance2, online);
    ev.setState(llcSegment2.getSegmentName(), helixInstance2, consuming);
    routingTable.markDataResourceOnline(resourceName, ev, instanceConfigs);
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resourceName, Collections.<String>emptyList());
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
    }
    // Now change the percent value in the routing table selector to be 100, and we should get only LLC segments.
    Configuration configuration = new PropertiesConfiguration();
    configuration.addProperty("class", PercentageBasedRoutingTableSelector.class.getName());
    configuration.addProperty("table." + resourceName, new Integer(100));
    RoutingTableSelector selector = RoutingTableSelectorFactory.getRoutingTableSelector(configuration, null);
    selector.init(configuration, null);
    Field selectorField = HelixExternalViewBasedRouting.class.getDeclaredField("_routingTableSelector");
    selectorField.setAccessible(true);
    selectorField.set(routingTable, selector);
    // And we should find only LLC segments.
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
    }
    // Now change it to 50, and we should find both (at least 10 times each).
    configuration = new PropertiesConfiguration();
    configuration.addProperty("table." + resourceName, new Integer(50));
    selector = new PercentageBasedRoutingTableSelector();
    selector.init(configuration, null);
    selectorField.set(routingTable, selector);
    int hlc = 0;
    int llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    // If we do the above iteration 100 times, we should get at least 10 of each type of routing.
    // If this test fails
    Assert.assertTrue(hlc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    Assert.assertTrue(llc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    // Check that force HLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_HLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 100);
    Assert.assertEquals(llc, 0);
    // Check that force LLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_LLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 0);
    Assert.assertEquals(llc, 100);
}

Example 29 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ScatterGatherImpl method buildInvertedMap.

/**
   * Merge segment-sets which have the same set of servers. If 2 segmentIds have overlapping
   * set of servers, they are not merged. If there is predefined-selection for a segmentId,
   * a separate entry is added for those in the inverted map.
   * @param requestContext Scatter gather request
   */
protected void buildInvertedMap(ScatterGatherRequestContext requestContext) {
    ScatterGatherRequest request = requestContext.getRequest();
    Map<ServerInstance, SegmentIdSet> segmentIdToInstanceMap = request.getSegmentsServicesMap();
    Map<List<ServerInstance>, SegmentIdSet> instanceToSegmentMap = new HashMap<List<ServerInstance>, SegmentIdSet>();
    for (ServerInstance serverInstance : segmentIdToInstanceMap.keySet()) {
        instanceToSegmentMap.put(Arrays.asList(serverInstance), segmentIdToInstanceMap.get(serverInstance));
    }
    requestContext.setInvertedMap(instanceToSegmentMap);
}

Example 30 with ServerInstance

use of com.linkedin.pinot.common.response.ServerInstance in project pinot by linkedin.

the class ScatterGatherImpl method selectServicesPerPartitionGroup.

/**
   * For each segment-set in the instanceToSegmentMap, we select one (or more speculative) servers
   *
   * @param requestContext
   */
private void selectServicesPerPartitionGroup(ScatterGatherRequestContext requestContext) {
    Map<ServerInstance, SegmentIdSet> selectedServers = new HashMap<ServerInstance, SegmentIdSet>();
    ScatterGatherRequest request = requestContext.getRequest();
    Map<List<ServerInstance>, SegmentIdSet> instanceToSegmentMap = requestContext.getInvertedMap();
    //int numDuplicateRequests = request.getNumSpeculativeRequests();
    ReplicaSelection selection = request.getReplicaSelection();
    for (Entry<List<ServerInstance>, SegmentIdSet> e : instanceToSegmentMap.entrySet()) {
        ServerInstance s = selection.selectServer(e.getValue().getOneSegment(), e.getKey(), request.getHashKey());
        mergePartitionGroup(selectedServers, s, e.getValue());
    /**
       * TODO:
       * We can easily add speculative execution here. The below code will pick a distinct server
       * for the segmentId, This entry needs to be maintained in a separate container in ScatterGatherRequestContext
       * Then in sndRequest, we need to construct SelectingFuture for the pairs of Future corresponding to original
       * and speculative(duplicate) request.
       *
       int numServers = e.getKey().size();

      // Pick Unique servers for speculative request
      int numIterations = Math.min(numServers - 1, numDuplicateRequests);
      for (int i = 0, c = 0; i < numIterations; i++, c++)
      {
        ServerInstance s1 = e.getKey().get(c);
        if ( s.equals(s1))
        {
          c++;
          s1 = e.getKey().get(c);
        }
        mergePartitionGroup(selectedServers, s1, e.getValue());
        //TODO: speculative servers need to be maintained in a separate entry in ScatterGatherRequestContext
      }
       **/
    }
    requestContext.setSelectedServers(selectedServers);
}