Examples with RoutingTableBuilder com.linkedin.pinot.routing.builder.RoutingTableBuilder used on opensource projects

Example 1 with RoutingTableBuilder

use of com.linkedin.pinot.routing.builder.RoutingTableBuilder in project pinot by linkedin.

the class RoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_OFFLINE");
    externalView.setState("segment0", "dataServer_instance_0", "ONLINE");
    externalView.setState("segment0", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_0", "ONLINE");
    List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2);
    routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs);
    ExternalView externalView1 = new ExternalView("testResource1_OFFLINE");
    externalView1.setState("segment10", "dataServer_instance_0", "ONLINE");
    externalView1.setState("segment11", "dataServer_instance_1", "ONLINE");
    externalView1.setState("segment12", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs);
    ExternalView externalView2 = new ExternalView("testResource2_OFFLINE");
    externalView2.setState("segment20", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs);
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3);
    }
}

Example 2 with RoutingTableBuilder

use of com.linkedin.pinot.routing.builder.RoutingTableBuilder in project pinot by linkedin.

the class RoutingTableTest method testKafkaHighLevelConsumerBasedRoutingTable.

@Test
public void testKafkaHighLevelConsumerBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new KafkaHighLevelConsumerBasedRoutingTableBuilder();
    final String group0 = "testResource0_REALTIME_1433316466991_0";
    final String group1 = "testResource1_REALTIME_1433316490099_1";
    final String group2 = "testResource2_REALTIME_1436589344583_1";
    final LLCSegmentName llcSegmentName = new LLCSegmentName("testResource0", 2, 65, System.currentTimeMillis());
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    Field realtimeRTBField = HelixExternalViewBasedRouting.class.getDeclaredField("_realtimeHLCRoutingTableBuilder");
    realtimeRTBField.setAccessible(true);
    realtimeRTBField.set(routingTable, routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_REALTIME");
    // Toss in an llc segment in the mix. Should not affect the results
    externalView.setState(llcSegmentName.getSegmentName(), "dataServer_instance_0", "CONSUMING");
    externalView.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "0").getSegmentName(), "dataServer_instance_0", "ONLINE");
    externalView.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "1").getSegmentName(), "dataServer_instance_1", "ONLINE");
    externalView.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "2").getSegmentName(), "dataServer_instance_2", "ONLINE");
    externalView.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "3").getSegmentName(), "dataServer_instance_3", "ONLINE");
    externalView.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "4").getSegmentName(), "dataServer_instance_4", "ONLINE");
    externalView.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "5").getSegmentName(), "dataServer_instance_5", "ONLINE");
    routingTable.markDataResourceOnline("testResource0_REALTIME", externalView, generateInstanceConfigs("dataServer_instance", 0, 5));
    ExternalView externalView1 = new ExternalView("testResource1_REALTIME");
    externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "10").getSegmentName(), "dataServer_instance_10", "ONLINE");
    externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "11").getSegmentName(), "dataServer_instance_11", "ONLINE");
    externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "12").getSegmentName(), "dataServer_instance_12", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_REALTIME", externalView1, generateInstanceConfigs("dataServer_instance", 10, 12));
    ExternalView externalView2 = new ExternalView("testResource2_REALTIME");
    externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "20").getSegmentName(), "dataServer_instance_20", "ONLINE");
    externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "21").getSegmentName(), "dataServer_instance_21", "ONLINE");
    externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "22").getSegmentName(), "dataServer_instance_22", "ONLINE");
    externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "23").getSegmentName(), "dataServer_instance_23", "ONLINE");
    externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "24").getSegmentName(), "dataServer_instance_24", "ONLINE");
    externalView2.setState(new HLCSegmentName(group1, ALL_PARTITIONS, "25").getSegmentName(), "dataServer_instance_25", "ONLINE");
    externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "26").getSegmentName(), "dataServer_instance_26", "ONLINE");
    externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "27").getSegmentName(), "dataServer_instance_27", "ONLINE");
    externalView2.setState(new HLCSegmentName(group2, ALL_PARTITIONS, "28").getSegmentName(), "dataServer_instance_28", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_REALTIME", externalView2, generateInstanceConfigs("dataServer_instance", 20, 28));
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "0").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "1").getSegmentName() + "]", "[" + new HLCSegmentName(group1, ALL_PARTITIONS, "2").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "3").getSegmentName() + "]", "[" + new HLCSegmentName(group2, ALL_PARTITIONS, "4").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "5").getSegmentName() + "]" }, 2);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "10").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "11").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "12").getSegmentName() + "]" }, 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_REALTIME", new String[] { "[" + new HLCSegmentName(group0, ALL_PARTITIONS, "20").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "21").getSegmentName() + ", " + new HLCSegmentName(group0, ALL_PARTITIONS, "22").getSegmentName() + "]", "[" + new HLCSegmentName(group1, ALL_PARTITIONS, "23").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "24").getSegmentName() + ", " + new HLCSegmentName(group1, ALL_PARTITIONS, "25").getSegmentName() + "]", "[" + new HLCSegmentName(group2, ALL_PARTITIONS, "26").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "27").getSegmentName() + ", " + new HLCSegmentName(group2, ALL_PARTITIONS, "28").getSegmentName() + "]" }, 3);
    }
}

Example 3 with RoutingTableBuilder

use of com.linkedin.pinot.routing.builder.RoutingTableBuilder in project pinot by linkedin.

the class HelixExternalViewBasedRouting method buildRoutingTable.

private void buildRoutingTable(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigs) {
    // Save the current version number of the external view to avoid unnecessary routing table updates
    int externalViewRecordVersion = externalView.getRecord().getVersion();
    _lastKnownExternalViewVersionMap.put(tableName, externalViewRecordVersion);
    RoutingTableBuilder routingTableBuilder;
    CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
    // Pick the appropriate routing table builder based on the table type
    if (CommonConstants.Helix.TableType.REALTIME.equals(tableType)) {
        routingTableBuilder = _realtimeHLCRoutingTableBuilder;
    } else {
        if (isLargeCluster(externalView)) {
            routingTableBuilder = _largeClusterRoutingTableBuilder;
        } else {
            routingTableBuilder = _smallClusterRoutingTableBuilder;
        }
    }
    LOGGER.info("Trying to compute routing table for table {} using {}", tableName, routingTableBuilder);
    long startTimeMillis = System.currentTimeMillis();
    try {
        Map<String, InstanceConfig> relevantInstanceConfigs = new HashMap<>();
        // Build a list of routing tables
        List<ServerToSegmentSetMap> serverToSegmentSetMap = routingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
        // Keep track of the instance configs that are used in that routing table
        updateInstanceConfigsMapFromRoutingTables(relevantInstanceConfigs, instanceConfigs, serverToSegmentSetMap);
        _brokerRoutingTable.put(tableName, serverToSegmentSetMap);
        // If this is a realtime table, also build a LLC routing table
        if (CommonConstants.Helix.TableType.REALTIME.equals(tableType)) {
            _routingTableSelector.registerTable(tableName);
            try {
                // Build the routing table
                List<ServerToSegmentSetMap> llcserverToSegmentSetMap = _realtimeLLCRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
                // Keep track of the instance configs that are used in that routing table
                updateInstanceConfigsMapFromRoutingTables(relevantInstanceConfigs, instanceConfigs, llcserverToSegmentSetMap);
                _llcBrokerRoutingTable.put(tableName, llcserverToSegmentSetMap);
            } catch (Exception e) {
                LOGGER.error("Failed to compute LLC routing table for {}. Ignoring", tableName, e);
            }
        }
        // Save the instance configs used so that we can avoid unnecessary routing table updates later
        _lastKnownInstanceConfigsForTable.put(tableName, relevantInstanceConfigs);
        for (InstanceConfig instanceConfig : relevantInstanceConfigs.values()) {
            _lastKnownInstanceConfigs.put(instanceConfig.getInstanceName(), instanceConfig);
        }
        // Ensure this table is registered with all relevant instances
        for (String instanceName : relevantInstanceConfigs.keySet()) {
            Set<String> tablesForCurrentInstance = _tablesForInstance.get(instanceName);
            // Ensure there is a table set for this instance
            if (tablesForCurrentInstance == null) {
                synchronized (_tablesForInstance) {
                    if (!_tablesForInstance.containsKey(instanceName)) {
                        tablesForCurrentInstance = Sets.newConcurrentHashSet();
                        _tablesForInstance.put(instanceName, tablesForCurrentInstance);
                    } else {
                        // Another thread has created a table set for this instance, use it
                        tablesForCurrentInstance = _tablesForInstance.get(instanceName);
                    }
                }
            }
            // Add the table to the set of tables for this instance
            tablesForCurrentInstance.add(tableName);
        }
    } catch (Exception e) {
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ROUTING_TABLE_REBUILD_FAILURES, 1L);
        LOGGER.error("Failed to compute/update the routing table", e);
        // Mark the routing table as needing a rebuild
        _lastKnownExternalViewVersionMap.put(tableName, INVALID_EXTERNAL_VIEW_VERSION);
    }
    try {
        // We need to compute the time boundary only in two situations:
        // 1) We're adding/updating an offline table and there's a realtime table that we're serving
        // 2) We're adding a new realtime table and there's already an offline table, in which case we need to update the
        //    time boundary for the existing offline table
        String tableForTimeBoundaryUpdate = null;
        ExternalView externalViewForTimeBoundaryUpdate = null;
        if (tableType == CommonConstants.Helix.TableType.OFFLINE) {
            // Does a realtime table exist?
            String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(TableNameBuilder.extractRawTableName(tableName));
            if (_brokerRoutingTable.containsKey(realtimeTableName)) {
                tableForTimeBoundaryUpdate = tableName;
                externalViewForTimeBoundaryUpdate = externalView;
            }
        }
        if (tableType == CommonConstants.Helix.TableType.REALTIME) {
            // Does an offline table exist?
            String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TableNameBuilder.extractRawTableName(tableName));
            if (_brokerRoutingTable.containsKey(offlineTableName)) {
                // Is there no time boundary?
                if (_timeBoundaryService.getTimeBoundaryInfoFor(offlineTableName) == null) {
                    tableForTimeBoundaryUpdate = offlineTableName;
                    externalViewForTimeBoundaryUpdate = fetchExternalView(offlineTableName);
                }
            }
        }
        if (tableForTimeBoundaryUpdate != null) {
            updateTimeBoundary(tableForTimeBoundaryUpdate, externalViewForTimeBoundaryUpdate);
        } else {
            LOGGER.info("No need to update time boundary for table {}", tableName);
        }
    } catch (Exception e) {
        LOGGER.error("Failed to update the TimeBoundaryService", e);
    }
    long updateTime = System.currentTimeMillis() - startTimeMillis;
    if (_brokerMetrics != null) {
        _brokerMetrics.addTimedValue(BrokerTimer.ROUTING_TABLE_UPDATE_TIME, updateTime, TimeUnit.MILLISECONDS);
    }
    LOGGER.info("Routing table update for table {} completed in {} ms", tableName, updateTime);
}

Example 4 with RoutingTableBuilder

use of com.linkedin.pinot.routing.builder.RoutingTableBuilder in project pinot by linkedin.

the class RandomRoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json");
    Assert.assertNotNull(resourceUrl);
    String fileName = resourceUrl.getFile();
    String tableName = "testTable_OFFLINE";
    InputStream evInputStream = new FileInputStream(fileName);
    ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
    ZNRecord externalViewRecord = (ZNRecord) znRecordSerializer.deserialize(IOUtils.toByteArray(evInputStream));
    int totalRuns = 10000;
    RoutingTableBuilder routingStrategy = new BalancedRandomRoutingTableBuilder(10);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, new PercentageBasedRoutingTableSelector(), null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView(externalViewRecord);
    routingTable.markDataResourceOnline(tableName, externalView, getInstanceConfigs(externalView));
    double[] globalArrays = new double[9];
    for (int numRun = 0; numRun < totalRuns; ++numRun) {
        RoutingTableLookupRequest request = new RoutingTableLookupRequest(tableName, Collections.<String>emptyList());
        Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
        TreeSet<ServerInstance> serverInstances = new TreeSet<ServerInstance>(serversMap.keySet());
        int i = 0;
        double[] arrays = new double[9];
        for (ServerInstance serverInstance : serverInstances) {
            globalArrays[i] += serversMap.get(serverInstance).getSegments().size();
            arrays[i++] = serversMap.get(serverInstance).getSegments().size();
        }
        for (int j = 0; i < arrays.length; ++j) {
            Assert.assertTrue(arrays[j] / totalRuns <= 31);
            Assert.assertTrue(arrays[j] / totalRuns >= 28);
        }
    //      System.out.println(Arrays.toString(arrays) + " : " + new StandardDeviation().evaluate(arrays) + " : " + new Mean().evaluate(arrays));
    }
    for (int i = 0; i < globalArrays.length; ++i) {
        Assert.assertTrue(globalArrays[i] / totalRuns <= 31);
        Assert.assertTrue(globalArrays[i] / totalRuns >= 28);
    }
//    System.out.println(Arrays.toString(globalArrays) + " : " + new StandardDeviation().evaluate(globalArrays) + " : "
//        + new Mean().evaluate(globalArrays));
}