Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 26 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method createExternalView.

private ExternalView createExternalView(String tableName, int segmentCount, int replicationFactor, int instanceCount) {
    ExternalView externalView = new ExternalView(tableName);
    String[] instanceNames = new String[instanceCount];
    for (int i = 0; i < instanceCount; i++) {
        instanceNames[i] = buildInstanceName(i);
    }
    int assignmentCount = 0;
    for (int i = 0; i < segmentCount; i++) {
        String segmentName = tableName + "_" + i;
        for (int j = 0; j < replicationFactor; j++) {
            externalView.setState(segmentName, instanceNames[assignmentCount % instanceCount], "ONLINE");
            ++assignmentCount;
        }
    }
    return externalView;
}

Example 27 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableServerLoadIsRelativelyEqual.

@Test
public void testRoutingTableServerLoadIsRelativelyEqual() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 300;
    final int replicationFactor = 10;
    final int instanceCount = 50;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    List<ServerToSegmentSetMap> routingTables = _largeClusterRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
    Map<String, Integer> segmentCountPerServer = new HashMap<>();
    // Count number of segments assigned per server
    for (ServerToSegmentSetMap routingTable : routingTables) {
        for (String server : routingTable.getServerSet()) {
            Integer segmentCountForServer = segmentCountPerServer.get(server);
            if (segmentCountForServer == null) {
                segmentCountForServer = 0;
            }
            segmentCountForServer += routingTable.getSegmentSet(server).size();
            segmentCountPerServer.put(server, segmentCountForServer);
        }
    }
    int minNumberOfSegmentsAssignedPerServer = Integer.MAX_VALUE;
    int maxNumberOfSegmentsAssignedPerServer = 0;
    for (Integer segmentCountForServer : segmentCountPerServer.values()) {
        if (segmentCountForServer < minNumberOfSegmentsAssignedPerServer) {
            minNumberOfSegmentsAssignedPerServer = segmentCountForServer;
        }
        if (maxNumberOfSegmentsAssignedPerServer < segmentCountForServer) {
            maxNumberOfSegmentsAssignedPerServer = segmentCountForServer;
        }
    }
    assertTrue(maxNumberOfSegmentsAssignedPerServer < minNumberOfSegmentsAssignedPerServer * 1.5, "At least one server has more than 150% of the load of the least loaded server, minNumberOfSegmentsAssignedPerServer = " + minNumberOfSegmentsAssignedPerServer + " maxNumberOfSegmentsAssignedPerServer = " + maxNumberOfSegmentsAssignedPerServer + " RANDOM_SEED = " + RANDOM_SEED);
}

Example 28 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixExternalViewBasedRouting method buildRoutingTable.

private void buildRoutingTable(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigs) {
    // Save the current version number of the external view to avoid unnecessary routing table updates
    int externalViewRecordVersion = externalView.getRecord().getVersion();
    _lastKnownExternalViewVersionMap.put(tableName, externalViewRecordVersion);
    RoutingTableBuilder routingTableBuilder;
    CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
    // Pick the appropriate routing table builder based on the table type
    if (CommonConstants.Helix.TableType.REALTIME.equals(tableType)) {
        routingTableBuilder = _realtimeHLCRoutingTableBuilder;
    } else {
        if (isLargeCluster(externalView)) {
            routingTableBuilder = _largeClusterRoutingTableBuilder;
        } else {
            routingTableBuilder = _smallClusterRoutingTableBuilder;
        }
    }
    LOGGER.info("Trying to compute routing table for table {} using {}", tableName, routingTableBuilder);
    long startTimeMillis = System.currentTimeMillis();
    try {
        Map<String, InstanceConfig> relevantInstanceConfigs = new HashMap<>();
        // Build a list of routing tables
        List<ServerToSegmentSetMap> serverToSegmentSetMap = routingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
        // Keep track of the instance configs that are used in that routing table
        updateInstanceConfigsMapFromRoutingTables(relevantInstanceConfigs, instanceConfigs, serverToSegmentSetMap);
        _brokerRoutingTable.put(tableName, serverToSegmentSetMap);
        // If this is a realtime table, also build a LLC routing table
        if (CommonConstants.Helix.TableType.REALTIME.equals(tableType)) {
            _routingTableSelector.registerTable(tableName);
            try {
                // Build the routing table
                List<ServerToSegmentSetMap> llcserverToSegmentSetMap = _realtimeLLCRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
                // Keep track of the instance configs that are used in that routing table
                updateInstanceConfigsMapFromRoutingTables(relevantInstanceConfigs, instanceConfigs, llcserverToSegmentSetMap);
                _llcBrokerRoutingTable.put(tableName, llcserverToSegmentSetMap);
            } catch (Exception e) {
                LOGGER.error("Failed to compute LLC routing table for {}. Ignoring", tableName, e);
            }
        }
        // Save the instance configs used so that we can avoid unnecessary routing table updates later
        _lastKnownInstanceConfigsForTable.put(tableName, relevantInstanceConfigs);
        for (InstanceConfig instanceConfig : relevantInstanceConfigs.values()) {
            _lastKnownInstanceConfigs.put(instanceConfig.getInstanceName(), instanceConfig);
        }
        // Ensure this table is registered with all relevant instances
        for (String instanceName : relevantInstanceConfigs.keySet()) {
            Set<String> tablesForCurrentInstance = _tablesForInstance.get(instanceName);
            // Ensure there is a table set for this instance
            if (tablesForCurrentInstance == null) {
                synchronized (_tablesForInstance) {
                    if (!_tablesForInstance.containsKey(instanceName)) {
                        tablesForCurrentInstance = Sets.newConcurrentHashSet();
                        _tablesForInstance.put(instanceName, tablesForCurrentInstance);
                    } else {
                        // Another thread has created a table set for this instance, use it
                        tablesForCurrentInstance = _tablesForInstance.get(instanceName);
                    }
                }
            }
            // Add the table to the set of tables for this instance
            tablesForCurrentInstance.add(tableName);
        }
    } catch (Exception e) {
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ROUTING_TABLE_REBUILD_FAILURES, 1L);
        LOGGER.error("Failed to compute/update the routing table", e);
        // Mark the routing table as needing a rebuild
        _lastKnownExternalViewVersionMap.put(tableName, INVALID_EXTERNAL_VIEW_VERSION);
    }
    try {
        // We need to compute the time boundary only in two situations:
        // 1) We're adding/updating an offline table and there's a realtime table that we're serving
        // 2) We're adding a new realtime table and there's already an offline table, in which case we need to update the
        //    time boundary for the existing offline table
        String tableForTimeBoundaryUpdate = null;
        ExternalView externalViewForTimeBoundaryUpdate = null;
        if (tableType == CommonConstants.Helix.TableType.OFFLINE) {
            // Does a realtime table exist?
            String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(TableNameBuilder.extractRawTableName(tableName));
            if (_brokerRoutingTable.containsKey(realtimeTableName)) {
                tableForTimeBoundaryUpdate = tableName;
                externalViewForTimeBoundaryUpdate = externalView;
            }
        }
        if (tableType == CommonConstants.Helix.TableType.REALTIME) {
            // Does an offline table exist?
            String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TableNameBuilder.extractRawTableName(tableName));
            if (_brokerRoutingTable.containsKey(offlineTableName)) {
                // Is there no time boundary?
                if (_timeBoundaryService.getTimeBoundaryInfoFor(offlineTableName) == null) {
                    tableForTimeBoundaryUpdate = offlineTableName;
                    externalViewForTimeBoundaryUpdate = fetchExternalView(offlineTableName);
                }
            }
        }
        if (tableForTimeBoundaryUpdate != null) {
            updateTimeBoundary(tableForTimeBoundaryUpdate, externalViewForTimeBoundaryUpdate);
        } else {
            LOGGER.info("No need to update time boundary for table {}", tableName);
        }
    } catch (Exception e) {
        LOGGER.error("Failed to update the TimeBoundaryService", e);
    }
    long updateTime = System.currentTimeMillis() - startTimeMillis;
    if (_brokerMetrics != null) {
        _brokerMetrics.addTimedValue(BrokerTimer.ROUTING_TABLE_UPDATE_TIME, updateTime, TimeUnit.MILLISECONDS);
    }
    LOGGER.info("Routing table update for table {} completed in {} ms", tableName, updateTime);
}

Example 29 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processExternalViewChange.

public void processExternalViewChange() {
    long startTime = System.currentTimeMillis();
    // Get list of tables that we're serving
    List<String> tablesServed = new ArrayList<>(_lastKnownExternalViewVersionMap.keySet());
    if (tablesServed.isEmpty()) {
        return;
    }
    // Build list of external views to fetch
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> externalViewPaths = new ArrayList<>(tablesServed.size());
    for (String tableName : tablesServed) {
        PropertyKey propertyKey = propertyKeyBuilder.externalView(tableName);
        externalViewPaths.add(propertyKey.getPath());
    }
    // Get znode stats for all tables that we're serving
    long statStartTime = System.currentTimeMillis();
    Stat[] externalViewStats = helixDataAccessor.getBaseDataAccessor().getStats(externalViewPaths, AccessOption.PERSISTENT);
    long statEndTime = System.currentTimeMillis();
    // Make a list of external views that changed
    List<String> tablesThatChanged = new ArrayList<>();
    long evCheckStartTime = System.currentTimeMillis();
    for (int i = 0; i < externalViewStats.length; i++) {
        Stat externalViewStat = externalViewStats[i];
        if (externalViewStat != null) {
            String currentTableName = tablesServed.get(i);
            int currentExternalViewVersion = externalViewStat.getVersion();
            int lastKnownExternalViewVersion = _lastKnownExternalViewVersionMap.get(currentTableName);
            if (lastKnownExternalViewVersion != currentExternalViewVersion) {
                tablesThatChanged.add(currentTableName);
            }
        }
    }
    long evCheckEndTime = System.currentTimeMillis();
    // Fetch the instance configs and update the routing tables for the tables that changed
    long icFetchTime = 0;
    long rebuildStartTime = System.currentTimeMillis();
    if (!tablesThatChanged.isEmpty()) {
        // Fetch instance configs
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableThatChanged : tablesThatChanged) {
            // We ignore the external views given by Helix on external view change and fetch the latest version as our
            // version of Helix (0.6.5) does not batch external view change messages.
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableThatChanged));
            buildRoutingTable(tableThatChanged, externalView, instanceConfigs);
        }
    }
    long rebuildEndTime = System.currentTimeMillis();
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed external view change in {} ms (stat {} ms, EV check {} ms, IC fetch {} ms, rebuild {} ms), routing tables rebuilt for tables {}, {} / {} routing tables rebuilt", (endTime - startTime), (statEndTime - statStartTime), (evCheckEndTime - evCheckStartTime), icFetchTime, (rebuildEndTime - rebuildStartTime), tablesThatChanged, tablesThatChanged.size(), tablesServed.size());
}

Example 30 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class RandomRoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json");
    Assert.assertNotNull(resourceUrl);
    String fileName = resourceUrl.getFile();
    String tableName = "testTable_OFFLINE";
    InputStream evInputStream = new FileInputStream(fileName);
    ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
    ZNRecord externalViewRecord = (ZNRecord) znRecordSerializer.deserialize(IOUtils.toByteArray(evInputStream));
    int totalRuns = 10000;
    RoutingTableBuilder routingStrategy = new BalancedRandomRoutingTableBuilder(10);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, new PercentageBasedRoutingTableSelector(), null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView(externalViewRecord);
    routingTable.markDataResourceOnline(tableName, externalView, getInstanceConfigs(externalView));
    double[] globalArrays = new double[9];
    for (int numRun = 0; numRun < totalRuns; ++numRun) {
        RoutingTableLookupRequest request = new RoutingTableLookupRequest(tableName, Collections.<String>emptyList());
        Map<ServerInstance, SegmentIdSet> serversMap = routingTable.findServers(request);
        TreeSet<ServerInstance> serverInstances = new TreeSet<ServerInstance>(serversMap.keySet());
        int i = 0;
        double[] arrays = new double[9];
        for (ServerInstance serverInstance : serverInstances) {
            globalArrays[i] += serversMap.get(serverInstance).getSegments().size();
            arrays[i++] = serversMap.get(serverInstance).getSegments().size();
        }
        for (int j = 0; i < arrays.length; ++j) {
            Assert.assertTrue(arrays[j] / totalRuns <= 31);
            Assert.assertTrue(arrays[j] / totalRuns >= 28);
        }
    //      System.out.println(Arrays.toString(arrays) + " : " + new StandardDeviation().evaluate(arrays) + " : " + new Mean().evaluate(arrays));
    }
    for (int i = 0; i < globalArrays.length; ++i) {
        Assert.assertTrue(globalArrays[i] / totalRuns <= 31);
        Assert.assertTrue(globalArrays[i] / totalRuns >= 28);
    }
//    System.out.println(Arrays.toString(globalArrays) + " : " + new StandardDeviation().evaluate(globalArrays) + " : "
//        + new Mean().evaluate(globalArrays));
}