Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 1 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class PinotSegmentRebalancer method isStable.

/**
   * return true if IdealState = ExternalView
   * @return
   */
public int isStable(String tableName) {
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName);
    ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName);
    Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields();
    Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields();
    int numDiff = 0;
    for (String segment : mapFieldsIS.keySet()) {
        Map<String, String> mapIS = mapFieldsIS.get(segment);
        Map<String, String> mapEV = mapFieldsEV.get(segment);
        for (String server : mapIS.keySet()) {
            String state = mapIS.get(server);
            if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) {
                LOGGER.info("Mismatch: segment" + segment + " server:" + server + " state:" + state);
                numDiff = numDiff + 1;
            }
        }
    }
    return numDiff;
}

Example 2 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class VerifySegmentState method execute.

@Override
public boolean execute() throws Exception {
    ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);
    List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
    for (String resourceName : resourcesInCluster) {
        if (resourceName.startsWith(tablePrefix)) {
            IdealState resourceIdealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
            ExternalView resourceExternalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
            Map<String, Map<String, String>> mapFieldsFromIS = resourceIdealState.getRecord().getMapFields();
            Map<String, Map<String, String>> mapFieldsFromEV = resourceExternalView.getRecord().getMapFields();
            boolean error = false;
            if (mapFieldsFromIS.size() != mapFieldsFromEV.size()) {
                LOGGER.info("Table: {}, idealState size: {} does NOT match external view size: {}", resourceName, mapFieldsFromIS.size(), mapFieldsFromEV.size());
                error = true;
            }
            if (!mapFieldsFromIS.keySet().equals(mapFieldsFromEV.keySet())) {
                Set<String> idealStateKeys = mapFieldsFromIS.keySet();
                Set<String> externalViewKeys = mapFieldsFromEV.keySet();
                Sets.SetView<String> isToEVDiff = Sets.difference(idealStateKeys, externalViewKeys);
                for (String segmentName : isToEVDiff) {
                    LOGGER.info("Segment: {} is missing in external view, ideal state: {}", segmentName, mapFieldsFromIS.get(segmentName));
                }
                Sets.SetView<String> evToISDiff = Sets.difference(externalViewKeys, idealStateKeys);
                for (String segmentName : evToISDiff) {
                    LOGGER.error("Segment: {} is missing in ideal state, external view: {}", segmentName, mapFieldsFromEV.get(segmentName));
                }
                error = true;
            }
            for (Map.Entry<String, Map<String, String>> idealEntry : mapFieldsFromIS.entrySet()) {
                String segmentName = idealEntry.getKey();
                Map<String, String> segmentIdealState = idealEntry.getValue();
                // try to format consistently for tool based parsing
                if (!mapFieldsFromEV.containsKey(segmentName)) {
                    LOGGER.info("Segment: {} idealstate: {} is MISSING in external view: {}", segmentName, segmentIdealState, "");
                }
                Map<String, String> segmentExternalView = mapFieldsFromEV.get(segmentName);
                if (!segmentIdealState.equals(segmentExternalView)) {
                    LOGGER.info("Segment: {} idealstate: {} does NOT match external view: {}", segmentName, segmentIdealState, segmentExternalView);
                    error = true;
                }
            }
            LOGGER.info(resourceName + " = " + (error ? "ERROR" : "OK"));
        }
    }
    return true;
}

Example 3 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class TimeBoundaryServiceTest method constructExternalView.

private ExternalView constructExternalView(String tableName) {
    ExternalView externalView = new ExternalView(tableName);
    List<OfflineSegmentZKMetadata> offlineResourceZKMetadataListForResource = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(_propertyStore, tableName);
    for (OfflineSegmentZKMetadata segmentMetadata : offlineResourceZKMetadataListForResource) {
        externalView.setState(segmentMetadata.getSegmentName(), "localhost", "ONLINE");
    }
    return externalView;
}

Example 4 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 5 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class RoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_OFFLINE");
    externalView.setState("segment0", "dataServer_instance_0", "ONLINE");
    externalView.setState("segment0", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_0", "ONLINE");
    List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2);
    routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs);
    ExternalView externalView1 = new ExternalView("testResource1_OFFLINE");
    externalView1.setState("segment10", "dataServer_instance_0", "ONLINE");
    externalView1.setState("segment11", "dataServer_instance_1", "ONLINE");
    externalView1.setState("segment12", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs);
    ExternalView externalView2 = new ExternalView("testResource2_OFFLINE");
    externalView2.setState("segment20", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs);
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3);
    }
}