Examples with RebalanceType org.apache.helix.api.config.StateTransitionThrottleConfig.RebalanceType used on opensource projects

Example 1 with RebalanceType

use of org.apache.helix.api.config.StateTransitionThrottleConfig.RebalanceType in project helix by apache.

the class IntermediateStateCalcStage method computeIntermediatePartitionState.

private PartitionStateMap computeIntermediatePartitionState(ClusterDataCache cache, ClusterStatusMonitor clusterStatusMonitor, IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput, PartitionStateMap bestPossiblePartitionStateMap, Map<String, List<String>> preferenceLists, StateTransitionThrottleController throttleController) {
    String resourceName = resource.getResourceName();
    logger.debug("Processing resource:" + resourceName);
    if (!throttleController.isThrottleEnabled() || !IdealState.RebalanceMode.FULL_AUTO.equals(idealState.getRebalanceMode()) || cache.isTaskCache()) {
        // We only apply throttling on FULL-AUTO now.
        return bestPossiblePartitionStateMap;
    }
    String stateModelDefName = idealState.getStateModelDefRef();
    StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
    PartitionStateMap intermediatePartitionStateMap = new PartitionStateMap(resourceName);
    Set<Partition> partitionsNeedRecovery = new HashSet<>();
    Set<Partition> partitionsNeedLoadbalance = new HashSet<>();
    Set<Partition> partitionshaveErrorStateReplica = new HashSet<>();
    for (Partition partition : resource.getPartitions()) {
        Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName, partition);
        Map<String, String> bestPossibleMap = bestPossiblePartitionStateMap.getPartitionMap(partition);
        List<String> preferenceList = preferenceLists.get(partition.getPartitionName());
        RebalanceType rebalanceType = getRebalanceType(cache, bestPossibleMap, preferenceList, stateModelDef, currentStateMap, idealState);
        // TODO refine getRebalanceType to return more accurate rebalance types.
        // So following logic doesn't need to check for more details.
        boolean rebalanceNeeded = false;
        if (rebalanceType.equals(RebalanceType.RECOVERY_BALANCE)) {
            // Check if any error exist
            if (currentStateMap.values().contains(HelixDefinedState.ERROR.name())) {
                partitionshaveErrorStateReplica.add(partition);
            }
            // Check if recovery is needed for this partition
            if (!currentStateMap.equals(bestPossibleMap)) {
                partitionsNeedRecovery.add(partition);
                rebalanceNeeded = true;
            }
        // else, if currentState == bestPossibleState, no rebalance needed
        } else if (rebalanceType.equals(RebalanceType.LOAD_BALANCE)) {
            partitionsNeedLoadbalance.add(partition);
            rebalanceNeeded = true;
        }
        if (!rebalanceNeeded) {
            // no rebalance needed.
            Map<String, String> intermediateMap = new HashMap<>(bestPossibleMap);
            intermediatePartitionStateMap.setState(partition, intermediateMap);
        }
    }
    if (!partitionsNeedRecovery.isEmpty()) {
        logger.info("recovery balance needed for " + resourceName + " partitions: " + partitionsNeedRecovery);
    }
    if (!partitionsNeedLoadbalance.isEmpty()) {
        logger.info("load balance needed for " + resourceName + " partitions: " + partitionsNeedLoadbalance);
    }
    if (!partitionshaveErrorStateReplica.isEmpty()) {
        logger.info("partition currently has ERROR replica in " + resourceName + " partitions: " + partitionshaveErrorStateReplica);
    }
    chargePendingTransition(resource, currentStateOutput, throttleController, partitionsNeedRecovery, partitionsNeedLoadbalance);
    // perform recovery rebalance
    Set<Partition> recoveryThrottledPartitions = recoveryRebalance(resource, bestPossiblePartitionStateMap, throttleController, intermediatePartitionStateMap, partitionsNeedRecovery, currentStateOutput, cache.getStateModelDef(resource.getStateModelDefRef()).getTopState());
    Set<Partition> loadbalanceThrottledPartitions = partitionsNeedLoadbalance;
    long maxAllowedErrorPartitions = cache.getClusterConfig().getErrorPartitionThresholdForLoadBalance();
    if (partitionsNeedRecovery.isEmpty() && (maxAllowedErrorPartitions < 0 || partitionshaveErrorStateReplica.size() <= maxAllowedErrorPartitions)) {
        // perform load balance only if
        // 1. no recovery operation to be scheduled.
        // 2. error partition count is less than configured limitation.
        loadbalanceThrottledPartitions = loadRebalance(resource, currentStateOutput, bestPossiblePartitionStateMap, throttleController, intermediatePartitionStateMap, partitionsNeedLoadbalance, currentStateOutput.getCurrentStateMap(resourceName));
    } else {
        // skip load balance, use current state mapping
        for (Partition p : partitionsNeedLoadbalance) {
            Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName, p);
            intermediatePartitionStateMap.setState(p, currentStateMap);
        }
    }
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.updateRebalancerStats(resourceName, partitionsNeedRecovery.size(), partitionsNeedLoadbalance.size(), recoveryThrottledPartitions.size(), loadbalanceThrottledPartitions.size());
    }
    if (logger.isDebugEnabled()) {
        logParitionMapState(resourceName, new HashSet<>(resource.getPartitions()), partitionsNeedRecovery, recoveryThrottledPartitions, partitionsNeedLoadbalance, loadbalanceThrottledPartitions, currentStateOutput, bestPossiblePartitionStateMap, intermediatePartitionStateMap);
    }
    logger.debug("End processing resource:" + resourceName);
    return intermediatePartitionStateMap;
}

Example 2 with RebalanceType

use of org.apache.helix.api.config.StateTransitionThrottleConfig.RebalanceType in project helix by apache.

the class IntermediateStateCalcStage method chargePendingTransition.

/**
 * Check and charge all pending transitions for throttling.
 */
private void chargePendingTransition(Resource resource, CurrentStateOutput currentStateOutput, StateTransitionThrottleController throttleController, Set<Partition> partitionsNeedRecovery, Set<Partition> partitionsNeedLoadbalance) {
    String resourceName = resource.getResourceName();
    // check and charge pending transitions
    for (Partition partition : resource.getPartitions()) {
        Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName, partition);
        Map<String, String> pendingMap = currentStateOutput.getPendingStateMap(resourceName, partition);
        StateTransitionThrottleConfig.RebalanceType rebalanceType = RebalanceType.NONE;
        if (partitionsNeedRecovery.contains(partition)) {
            rebalanceType = StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE;
        } else if (partitionsNeedLoadbalance.contains(partition)) {
            rebalanceType = StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE;
        }
        if (pendingMap.size() > 0) {
            throttleController.chargeCluster(rebalanceType);
            throttleController.chargeResource(rebalanceType, resourceName);
            // charge each instance.
            for (String ins : pendingMap.keySet()) {
                String currentState = currentStateMap.get(ins);
                String pendingState = pendingMap.get(ins);
                if (pendingState != null && !pendingState.equals(currentState)) {
                    throttleController.chargeInstance(rebalanceType, ins);
                }
            }
        }
    }
}