Examples with PartitionStateMap org.apache.helix.controller.common.PartitionStateMap used on opensource projects

Example 1 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class IntermediateStateCalcStage method computeIntermediatePartitionState.

private PartitionStateMap computeIntermediatePartitionState(ClusterDataCache cache, ClusterStatusMonitor clusterStatusMonitor, IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput, PartitionStateMap bestPossiblePartitionStateMap, Map<String, List<String>> preferenceLists, StateTransitionThrottleController throttleController) {
    String resourceName = resource.getResourceName();
    logger.debug("Processing resource:" + resourceName);
    if (!throttleController.isThrottleEnabled() || !IdealState.RebalanceMode.FULL_AUTO.equals(idealState.getRebalanceMode()) || cache.isTaskCache()) {
        // We only apply throttling on FULL-AUTO now.
        return bestPossiblePartitionStateMap;
    }
    String stateModelDefName = idealState.getStateModelDefRef();
    StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
    PartitionStateMap intermediatePartitionStateMap = new PartitionStateMap(resourceName);
    Set<Partition> partitionsNeedRecovery = new HashSet<>();
    Set<Partition> partitionsNeedLoadbalance = new HashSet<>();
    Set<Partition> partitionshaveErrorStateReplica = new HashSet<>();
    for (Partition partition : resource.getPartitions()) {
        Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName, partition);
        Map<String, String> bestPossibleMap = bestPossiblePartitionStateMap.getPartitionMap(partition);
        List<String> preferenceList = preferenceLists.get(partition.getPartitionName());
        RebalanceType rebalanceType = getRebalanceType(cache, bestPossibleMap, preferenceList, stateModelDef, currentStateMap, idealState);
        // TODO refine getRebalanceType to return more accurate rebalance types.
        // So following logic doesn't need to check for more details.
        boolean rebalanceNeeded = false;
        if (rebalanceType.equals(RebalanceType.RECOVERY_BALANCE)) {
            // Check if any error exist
            if (currentStateMap.values().contains(HelixDefinedState.ERROR.name())) {
                partitionshaveErrorStateReplica.add(partition);
            }
            // Check if recovery is needed for this partition
            if (!currentStateMap.equals(bestPossibleMap)) {
                partitionsNeedRecovery.add(partition);
                rebalanceNeeded = true;
            }
        // else, if currentState == bestPossibleState, no rebalance needed
        } else if (rebalanceType.equals(RebalanceType.LOAD_BALANCE)) {
            partitionsNeedLoadbalance.add(partition);
            rebalanceNeeded = true;
        }
        if (!rebalanceNeeded) {
            // no rebalance needed.
            Map<String, String> intermediateMap = new HashMap<>(bestPossibleMap);
            intermediatePartitionStateMap.setState(partition, intermediateMap);
        }
    }
    if (!partitionsNeedRecovery.isEmpty()) {
        logger.info("recovery balance needed for " + resourceName + " partitions: " + partitionsNeedRecovery);
    }
    if (!partitionsNeedLoadbalance.isEmpty()) {
        logger.info("load balance needed for " + resourceName + " partitions: " + partitionsNeedLoadbalance);
    }
    if (!partitionshaveErrorStateReplica.isEmpty()) {
        logger.info("partition currently has ERROR replica in " + resourceName + " partitions: " + partitionshaveErrorStateReplica);
    }
    chargePendingTransition(resource, currentStateOutput, throttleController, partitionsNeedRecovery, partitionsNeedLoadbalance);
    // perform recovery rebalance
    Set<Partition> recoveryThrottledPartitions = recoveryRebalance(resource, bestPossiblePartitionStateMap, throttleController, intermediatePartitionStateMap, partitionsNeedRecovery, currentStateOutput, cache.getStateModelDef(resource.getStateModelDefRef()).getTopState());
    Set<Partition> loadbalanceThrottledPartitions = partitionsNeedLoadbalance;
    long maxAllowedErrorPartitions = cache.getClusterConfig().getErrorPartitionThresholdForLoadBalance();
    if (partitionsNeedRecovery.isEmpty() && (maxAllowedErrorPartitions < 0 || partitionshaveErrorStateReplica.size() <= maxAllowedErrorPartitions)) {
        // perform load balance only if
        // 1. no recovery operation to be scheduled.
        // 2. error partition count is less than configured limitation.
        loadbalanceThrottledPartitions = loadRebalance(resource, currentStateOutput, bestPossiblePartitionStateMap, throttleController, intermediatePartitionStateMap, partitionsNeedLoadbalance, currentStateOutput.getCurrentStateMap(resourceName));
    } else {
        // skip load balance, use current state mapping
        for (Partition p : partitionsNeedLoadbalance) {
            Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(resourceName, p);
            intermediatePartitionStateMap.setState(p, currentStateMap);
        }
    }
    if (clusterStatusMonitor != null) {
        clusterStatusMonitor.updateRebalancerStats(resourceName, partitionsNeedRecovery.size(), partitionsNeedLoadbalance.size(), recoveryThrottledPartitions.size(), loadbalanceThrottledPartitions.size());
    }
    if (logger.isDebugEnabled()) {
        logParitionMapState(resourceName, new HashSet<>(resource.getPartitions()), partitionsNeedRecovery, recoveryThrottledPartitions, partitionsNeedLoadbalance, loadbalanceThrottledPartitions, currentStateOutput, bestPossiblePartitionStateMap, intermediatePartitionStateMap);
    }
    logger.debug("End processing resource:" + resourceName);
    return intermediatePartitionStateMap;
}

Example 2 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class TestRebalancerMetrics method copyCurrentStateFromBestPossible.

private CurrentStateOutput copyCurrentStateFromBestPossible(BestPossibleStateOutput bestPossibleStateOutput, String resource) {
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    PartitionStateMap partitionStateMap = bestPossibleStateOutput.getPartitionStateMap(resource);
    for (Partition partition : partitionStateMap.partitionSet()) {
        Map<String, String> stateMap = partitionStateMap.getPartitionMap(partition);
        for (String instance : stateMap.keySet()) {
            currentStateOutput.setCurrentState(resource, partition, instance, stateMap.get(instance));
        }
    }
    return currentStateOutput;
}

Example 3 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStatesForRecoveryBalance.

private void updateCurrentStatesForRecoveryBalance(List<String> resourcePriority, CurrentStateOutput currentStateOutput) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    for (PartitionStateMap partitionStateMap : output.getResourceStatesMap().values()) {
        String resourceName = partitionStateMap.getResourceName();
        Partition partition = new Partition(resourceName + "_0");
        String instanceName = HOSTNAME_PREFIX + resourceName.split("_")[1];
        if (partitionStateMap.getPartitionMap(partition).values().contains("SLAVE") && !resourcePriority.contains(resourceName)) {
            updateCurrentOutput(resourcePriority, currentStateOutput, resourceName, partition, instanceName, "SLAVE");
            break;
        }
    }
}

Example 4 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStatesForLoadBalance.

private void updateCurrentStatesForLoadBalance(List<String> resourcePriority, CurrentStateOutput currentStateOutput) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    for (PartitionStateMap partitionStateMap : output.getResourceStatesMap().values()) {
        String resourceName = partitionStateMap.getResourceName();
        Partition partition = new Partition(resourceName + "_0");
        String oldInstance = HOSTNAME_PREFIX + resourceName.split("_")[1];
        String expectedInstance = HOSTNAME_PREFIX + (Integer.parseInt(resourceName.split("_")[1]) + 1);
        if (partitionStateMap.getPartitionMap(partition).containsKey(expectedInstance) && !resourcePriority.contains(resourceName)) {
            currentStateOutput.getCurrentStateMap(resourceName, partition).remove(oldInstance);
            updateCurrentOutput(resourcePriority, currentStateOutput, resourceName, partition, expectedInstance, "MASTER");
            break;
        }
    }
}

Example 5 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class BestPossibleExternalViewVerifier method verifyState.

@Override
protected synchronized boolean verifyState() {
    try {
        PropertyKey.Builder keyBuilder = _accessor.keyBuilder();
        // read cluster once and do verification
        ClusterDataCache cache = new ClusterDataCache();
        cache.refresh(_accessor);
        Map<String, IdealState> idealStates = cache.getIdealStates();
        if (idealStates == null) {
            // ideal state is null because ideal state is dropped
            idealStates = Collections.emptyMap();
        }
        // filter out all resources that use Task state model
        Iterator<Map.Entry<String, IdealState>> it = idealStates.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, IdealState> pair = it.next();
            if (pair.getValue().getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME)) {
                it.remove();
            }
        }
        // verify live instances.
        if (_expectLiveInstances != null && !_expectLiveInstances.isEmpty()) {
            Set<String> actualLiveNodes = cache.getLiveInstances().keySet();
            if (!_expectLiveInstances.equals(actualLiveNodes)) {
                LOG.warn("Live instances are not as expected. Actual live nodes: " + actualLiveNodes.toString());
                return false;
            }
        }
        Map<String, ExternalView> extViews = _accessor.getChildValuesMap(keyBuilder.externalViews());
        if (extViews == null) {
            extViews = Collections.emptyMap();
        }
        // Filter resources if requested
        if (_resources != null && !_resources.isEmpty()) {
            idealStates.keySet().retainAll(_resources);
            extViews.keySet().retainAll(_resources);
        }
        // add empty idealState for the resource
        for (String resource : extViews.keySet()) {
            if (!idealStates.containsKey(resource)) {
                ExternalView ev = extViews.get(resource);
                IdealState is = new IdealState(resource);
                is.getRecord().setSimpleFields(ev.getRecord().getSimpleFields());
                idealStates.put(resource, is);
            }
        }
        // calculate best possible state
        BestPossibleStateOutput bestPossOutput = calcBestPossState(cache);
        Map<String, Map<Partition, Map<String, String>>> bestPossStateMap = bestPossOutput.getStateMap();
        // set error states
        if (_errStates != null) {
            for (String resourceName : _errStates.keySet()) {
                Map<String, String> partErrStates = _errStates.get(resourceName);
                for (String partitionName : partErrStates.keySet()) {
                    String instanceName = partErrStates.get(partitionName);
                    if (!bestPossStateMap.containsKey(resourceName)) {
                        bestPossStateMap.put(resourceName, new HashMap<Partition, Map<String, String>>());
                    }
                    Partition partition = new Partition(partitionName);
                    if (!bestPossStateMap.get(resourceName).containsKey(partition)) {
                        bestPossStateMap.get(resourceName).put(partition, new HashMap<String, String>());
                    }
                    bestPossStateMap.get(resourceName).get(partition).put(instanceName, HelixDefinedState.ERROR.toString());
                }
            }
        }
        for (String resourceName : idealStates.keySet()) {
            ExternalView extView = extViews.get(resourceName);
            IdealState is = idealStates.get(resourceName);
            if (extView == null) {
                if (is.isExternalViewDisabled()) {
                    continue;
                } else {
                    LOG.error("externalView for " + resourceName + " is not available");
                    return false;
                }
            }
            // step 0: remove empty map and DROPPED state from best possible state
            PartitionStateMap bpStateMap = bestPossOutput.getPartitionStateMap(resourceName);
            StateModelDefinition stateModelDef = cache.getStateModelDef(is.getStateModelDefRef());
            if (stateModelDef == null) {
                LOG.error("State model definition " + is.getStateModelDefRef() + " for resource not found!" + is.getResourceName());
                return false;
            }
            boolean result = verifyExternalView(extView, bpStateMap, stateModelDef);
            if (!result) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("verifyExternalView fails for " + resourceName + "! ExternalView: " + extView + " BestPossibleState: " + bpStateMap);
                } else {
                    LOG.warn("verifyExternalView fails for " + resourceName + "! ExternalView does not match BestPossibleState");
                }
                return false;
            }
        }
        return true;
    } catch (Exception e) {
        LOG.error("exception in verification", e);
        return false;
    }
}