Examples with PropertyKey org.apache.helix.PropertyKey used on opensource projects

Example 6 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class ShowClusterInfoCommand method execute.

@Override
public boolean execute() throws Exception {
    Set<String> includeTableSet = new HashSet<>();
    String[] includeTables = _tables.split(",");
    for (String includeTable : includeTables) {
        String name = stripTypeFromName(includeTable.trim());
        if (name.length() > 0) {
            includeTableSet.add(name);
        }
    }
    Set<String> includeTagSet = new HashSet<>();
    String[] includeTags = _tags.split(",");
    for (String includeTag : includeTags) {
        String name = stripTypeFromName(includeTag.trim());
        if (name.length() > 0) {
            includeTagSet.add(name);
        }
    }
    ClusterInfo clusterInfo = new ClusterInfo();
    clusterInfo.clusterName = _clusterName;
    ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(_zkAddress);
    if (!zkHelixAdmin.getClusters().contains(_clusterName)) {
        LOGGER.error("Cluster {} not found in {}.", _clusterName, _zkAddress);
        return false;
    }
    List<String> instancesInCluster = zkHelixAdmin.getInstancesInCluster(_clusterName);
    List<String> tables = zkHelixAdmin.getResourcesInCluster(_clusterName);
    ZkClient zkClient = new ZkClient(_zkAddress);
    zkClient.setZkSerializer(new ZNRecordStreamingSerializer());
    LOGGER.info("Connecting to Zookeeper at: {}", _zkAddress);
    zkClient.waitUntilConnected();
    ZkBaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    ZKHelixDataAccessor zkHelixDataAccessor = new ZKHelixDataAccessor(_clusterName, baseDataAccessor);
    PropertyKey property = zkHelixDataAccessor.keyBuilder().liveInstances();
    List<String> liveInstances = zkHelixDataAccessor.getChildNames(property);
    PropertyKey controllerLeaderKey = zkHelixDataAccessor.keyBuilder().controllerLeader();
    LiveInstance controllerLeaderLiveInstance = zkHelixDataAccessor.getProperty(controllerLeaderKey);
    ControllerInfo controllerInfo = new ControllerInfo();
    controllerInfo.leaderName = controllerLeaderLiveInstance.getId();
    clusterInfo.controllerInfo = controllerInfo;
    for (String server : instancesInCluster) {
        if (server.startsWith("Server")) {
            ServerInfo serverInfo = new ServerInfo();
            serverInfo.name = server;
            serverInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE";
            InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server);
            serverInfo.tags = config.getRecord().getListField("TAG_LIST");
            clusterInfo.addServerInfo(serverInfo);
        }
        if (server.startsWith("Broker")) {
            BrokerInfo brokerInfo = new BrokerInfo();
            brokerInfo.name = server;
            brokerInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE";
            InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server);
            brokerInfo.tags = config.getRecord().getListField("TAG_LIST");
            clusterInfo.addBrokerInfo(brokerInfo);
        }
    }
    for (String table : tables) {
        if ("brokerResource".equalsIgnoreCase(table)) {
            continue;
        }
        TableInfo tableInfo = new TableInfo();
        IdealState idealState = zkHelixAdmin.getResourceIdealState(_clusterName, table);
        ExternalView externalView = zkHelixAdmin.getResourceExternalView(_clusterName, table);
        Set<String> segmentsFromIdealState = idealState.getPartitionSet();
        tableInfo.tableName = table;
        tableInfo.tag = idealState.getRecord().getSimpleField("INSTANCE_GROUP_TAG");
        String rawTableName = stripTypeFromName(tableInfo.tableName);
        String rawTagName = stripTypeFromName(tableInfo.tag);
        if (!includeTableSet.isEmpty() && !includeTableSet.contains(rawTableName)) {
            continue;
        }
        if (!includeTagSet.isEmpty() && !includeTagSet.contains(rawTagName)) {
            continue;
        }
        for (String segment : segmentsFromIdealState) {
            SegmentInfo segmentInfo = new SegmentInfo();
            segmentInfo.name = segment;
            Map<String, String> serverStateMapFromIS = idealState.getInstanceStateMap(segment);
            if (serverStateMapFromIS == null) {
                LOGGER.info("Unassigned segment {} in ideal state", segment);
                serverStateMapFromIS = Collections.emptyMap();
            }
            Map<String, String> serverStateMapFromEV = externalView.getStateMap(segment);
            if (serverStateMapFromEV == null) {
                LOGGER.info("Unassigned segment {} in external view", segment);
                serverStateMapFromEV = Collections.emptyMap();
            }
            for (String serverName : serverStateMapFromIS.keySet()) {
                segmentInfo.segmentStateMap.put(serverName, serverStateMapFromEV.get(serverName));
            }
            tableInfo.addSegmentInfo(segmentInfo);
        }
        clusterInfo.addTableInfo(tableInfo);
    }
    Yaml yaml = new Yaml();
    StringWriter sw = new StringWriter();
    yaml.dump(clusterInfo, sw);
    LOGGER.info(sw.toString());
    return true;
}

Example 7 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processExternalViewChange.

public void processExternalViewChange() {
    long startTime = System.currentTimeMillis();
    // Get list of tables that we're serving
    List<String> tablesServed = new ArrayList<>(_lastKnownExternalViewVersionMap.keySet());
    if (tablesServed.isEmpty()) {
        return;
    }
    // Build list of external views to fetch
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> externalViewPaths = new ArrayList<>(tablesServed.size());
    for (String tableName : tablesServed) {
        PropertyKey propertyKey = propertyKeyBuilder.externalView(tableName);
        externalViewPaths.add(propertyKey.getPath());
    }
    // Get znode stats for all tables that we're serving
    long statStartTime = System.currentTimeMillis();
    Stat[] externalViewStats = helixDataAccessor.getBaseDataAccessor().getStats(externalViewPaths, AccessOption.PERSISTENT);
    long statEndTime = System.currentTimeMillis();
    // Make a list of external views that changed
    List<String> tablesThatChanged = new ArrayList<>();
    long evCheckStartTime = System.currentTimeMillis();
    for (int i = 0; i < externalViewStats.length; i++) {
        Stat externalViewStat = externalViewStats[i];
        if (externalViewStat != null) {
            String currentTableName = tablesServed.get(i);
            int currentExternalViewVersion = externalViewStat.getVersion();
            int lastKnownExternalViewVersion = _lastKnownExternalViewVersionMap.get(currentTableName);
            if (lastKnownExternalViewVersion != currentExternalViewVersion) {
                tablesThatChanged.add(currentTableName);
            }
        }
    }
    long evCheckEndTime = System.currentTimeMillis();
    // Fetch the instance configs and update the routing tables for the tables that changed
    long icFetchTime = 0;
    long rebuildStartTime = System.currentTimeMillis();
    if (!tablesThatChanged.isEmpty()) {
        // Fetch instance configs
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableThatChanged : tablesThatChanged) {
            // We ignore the external views given by Helix on external view change and fetch the latest version as our
            // version of Helix (0.6.5) does not batch external view change messages.
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableThatChanged));
            buildRoutingTable(tableThatChanged, externalView, instanceConfigs);
        }
    }
    long rebuildEndTime = System.currentTimeMillis();
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed external view change in {} ms (stat {} ms, EV check {} ms, IC fetch {} ms, rebuild {} ms), routing tables rebuilt for tables {}, {} / {} routing tables rebuilt", (endTime - startTime), (statEndTime - statStartTime), (evCheckEndTime - evCheckStartTime), icFetchTime, (rebuildEndTime - rebuildStartTime), tablesThatChanged, tablesThatChanged.size(), tablesServed.size());
}

Example 8 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class PinotHelixResourceManager method toogleInstance.

/**
   * Toggle the status of an Instance between OFFLINE and ONLINE.
   * Keeps checking until ideal-state is successfully updated or times out.
   *
   * @param instanceName: Name of Instance for which the status needs to be toggled.
   * @param toggle: 'True' for ONLINE 'False' for OFFLINE.
   * @param timeOutInSeconds: Time-out for setting ideal-state.
   * @return
   */
public PinotResourceManagerResponse toogleInstance(String instanceName, boolean toggle, int timeOutInSeconds) {
    if (!instanceExists(instanceName)) {
        return new PinotResourceManagerResponse("Instance " + instanceName + " does not exist.", false);
    }
    _helixAdmin.enableInstance(_helixClusterName, instanceName, toggle);
    long deadline = System.currentTimeMillis() + 1000 * timeOutInSeconds;
    boolean toggleSucceed = false;
    String beforeToggleStates = (toggle) ? SegmentOnlineOfflineStateModel.OFFLINE : SegmentOnlineOfflineStateModel.ONLINE;
    while (System.currentTimeMillis() < deadline) {
        toggleSucceed = true;
        PropertyKey liveInstanceKey = _keyBuilder.liveInstance(instanceName);
        LiveInstance liveInstance = _helixDataAccessor.getProperty(liveInstanceKey);
        if (liveInstance == null) {
            if (toggle) {
                return PinotResourceManagerResponse.FAILURE_RESPONSE;
            } else {
                return PinotResourceManagerResponse.SUCCESS_RESPONSE;
            }
        }
        PropertyKey instanceCurrentStatesKey = _keyBuilder.currentStates(instanceName, liveInstance.getSessionId());
        List<CurrentState> instanceCurrentStates = _helixDataAccessor.getChildValues(instanceCurrentStatesKey);
        if (instanceCurrentStates == null) {
            return PinotResourceManagerResponse.SUCCESS_RESPONSE;
        } else {
            for (CurrentState currentState : instanceCurrentStates) {
                for (String state : currentState.getPartitionStateMap().values()) {
                    if (beforeToggleStates.equals(state)) {
                        toggleSucceed = false;
                    }
                }
            }
        }
        if (toggleSucceed) {
            return (toggle) ? new PinotResourceManagerResponse("Instance " + instanceName + " enabled.", true) : new PinotResourceManagerResponse("Instance " + instanceName + " disabled.", true);
        } else {
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            }
        }
    }
    return new PinotResourceManagerResponse("Instance enable/disable failed, timeout.", false);
}

Example 9 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class PinotHelixResourceManager method toggleSegmentState.

/**
   * Toggle the status of segment between ONLINE (enable = true) and OFFLINE (enable = FALSE).
   *
   * @param tableName: Name of table to which the segment belongs.
   * @param segments: List of segment for which to toggle the status.
   * @param enable: True for ONLINE, False for OFFLINE.
   * @param timeoutInSeconds Time out for toggling segment state.
   * @return
   */
public PinotResourceManagerResponse toggleSegmentState(String tableName, List<String> segments, boolean enable, long timeoutInSeconds) {
    String status = (enable) ? "ONLINE" : "OFFLINE";
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    boolean updateSuccessful;
    boolean externalViewUpdateSuccessful = true;
    long deadline = System.currentTimeMillis() + 1000 * timeoutInSeconds;
    // Set all partitions to offline to unload them from the servers
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        for (String segmentName : segments) {
            final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
            if (instanceSet == null || instanceSet.isEmpty()) {
                return new PinotResourceManagerResponse("Segment " + segmentName + " not found.", false);
            }
            for (final String instance : instanceSet) {
                idealState.setPartitionState(segmentName, instance, status);
            }
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful && (System.currentTimeMillis() <= deadline));
    // Check that the ideal state has been updated.
    LOGGER.info("Ideal state successfully updated, waiting to update external view");
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    for (String segmentName : segments) {
        Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
        for (String state : instanceStateMap.values()) {
            if (!status.equals(state)) {
                return new PinotResourceManagerResponse("Error: Failed to update Ideal state when setting status " + status + " for segment " + segmentName, false);
            }
        }
        // Wait until the partitions are offline in the external view
        if (!ifExternalViewChangeReflectedForState(tableName, segmentName, status, (timeoutInSeconds * 1000), true)) {
            externalViewUpdateSuccessful = false;
        }
    }
    return (externalViewUpdateSuccessful) ? new PinotResourceManagerResponse(("Success: Segment(s) " + " now " + status), true) : new PinotResourceManagerResponse("Error: Timed out. External view not completely updated", false);
}