Examples with StageException org.apache.helix.controller.pipeline.StageException used on opensource projects

Example 6 with StageException

use of org.apache.helix.controller.pipeline.StageException in project helix by apache.

the class ResourceValidationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache");
    }
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (resourceMap == null) {
        throw new StageException("Resources must be computed prior to validation!");
    }
    Map<String, IdealState> idealStateMap = cache.getIdealStates();
    Map<String, Map<String, String>> idealStateRuleMap = cache.getIdealStateRules();
    for (String resourceName : idealStateMap.keySet()) {
        // check every ideal state against the ideal state rules
        // the pipeline should not process any resources that have an unsupported ideal state
        IdealState idealState = idealStateMap.get(resourceName);
        if (!idealStateRuleMap.isEmpty()) {
            boolean hasMatchingRule = false;
            for (String ruleName : idealStateRuleMap.keySet()) {
                Map<String, String> rule = idealStateRuleMap.get(ruleName);
                boolean matches = idealStateMatchesRule(idealState, rule);
                hasMatchingRule = hasMatchingRule || matches;
                if (matches) {
                    break;
                }
            }
            if (!hasMatchingRule) {
                LOG.warn("Resource " + resourceName + " does not have a valid ideal state!");
                resourceMap.remove(resourceName);
            }
        }
        // check that every resource to process has a live state model definition
        String stateModelDefRef = idealState.getStateModelDefRef();
        StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefRef);
        if (stateModelDef == null) {
            LOG.warn("Resource " + resourceName + " uses state model " + stateModelDefRef + ", but it is not on the cluster!");
            resourceMap.remove(resourceName);
        }
    }
}

Example 7 with StageException

use of org.apache.helix.controller.pipeline.StageException in project helix by apache.

the class BestPossibleStateCalcStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    final Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    final ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (currentStateOutput == null || resourceMap == null || cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires CURRENT_STATE|RESOURCES|DataCache");
    }
    // Reset current INIT/RUNNING tasks on participants for throttling
    cache.resetActiveTaskCount(currentStateOutput);
    // Check whether the offline/disabled instance count in the cluster reaches the set limit,
    // if yes, pause the rebalancer.
    validateOfflineInstancesLimit(cache, (HelixManager) event.getAttribute(AttributeName.helixmanager.name()), clusterStatusMonitor);
    final BestPossibleStateOutput bestPossibleStateOutput = compute(event, resourceMap, currentStateOutput);
    event.addAttribute(AttributeName.BEST_POSSIBLE_STATE.name(), bestPossibleStateOutput);
    if (!cache.isTaskCache()) {
        final Map<String, InstanceConfig> instanceConfigMap = cache.getInstanceConfigMap();
        final Map<String, StateModelDefinition> stateModelDefMap = cache.getStateModelDefMap();
        asyncExecute(cache.getAsyncTasksThreadPool(), new Callable<Object>() {

            @Override
            public Object call() {
                try {
                    if (clusterStatusMonitor != null) {
                        clusterStatusMonitor.setPerInstanceResourceStatus(bestPossibleStateOutput, instanceConfigMap, resourceMap, stateModelDefMap);
                    }
                } catch (Exception e) {
                    logger.error("Could not update cluster status metrics!", e);
                }
                return null;
            }
        });
    }
}

Example 8 with StageException

use of org.apache.helix.controller.pipeline.StageException in project helix by apache.

the class CompatibilityCheckStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (manager == null || cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager | DataCache");
    }
    HelixManagerProperties properties = manager.getProperties();
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    for (LiveInstance liveInstance : liveInstanceMap.values()) {
        String participantVersion = liveInstance.getHelixVersion();
        if (!properties.isParticipantCompatible(participantVersion)) {
            String errorMsg = "incompatible participant. pipeline will not continue. " + "controller: " + manager.getInstanceName() + ", controllerVersion: " + properties.getVersion() + ", minimumSupportedParticipantVersion: " + properties.getProperty("miminum_supported_version.participant") + ", participant: " + liveInstance.getInstanceName() + ", participantVersion: " + participantVersion;
            LOG.error(errorMsg);
            throw new StageException(errorMsg);
        }
    }
}

Example 9 with StageException

use of org.apache.helix.controller.pipeline.StageException in project helix by apache.

the class CurrentStateComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (cache == null || resourceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (LiveInstance instance : liveInstances.values()) {
        String instanceName = instance.getInstanceName();
        String instanceSessionId = instance.getSessionId();
        // update pending messages
        Map<String, Message> messages = cache.getMessages(instanceName);
        updatePendingMessages(instance, messages.values(), currentStateOutput, resourceMap);
        // update current states.
        Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, instanceSessionId);
        updateCurrentStates(instance, currentStateMap.values(), currentStateOutput, resourceMap);
    }
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        updateMissingTopStateStatus(cache, clusterStatusMonitor, resourceMap, currentStateOutput);
    }
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
}

Example 10 with StageException

use of org.apache.helix.controller.pipeline.StageException in project helix by apache.

the class ResourceComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache");
    }
    Map<String, IdealState> idealStates = cache.getIdealStates();
    Map<String, Resource> resourceMap = new LinkedHashMap<String, Resource>();
    Map<String, Resource> resourceToRebalance = new LinkedHashMap<>();
    if (idealStates != null && idealStates.size() > 0) {
        for (IdealState idealState : idealStates.values()) {
            if (idealState == null) {
                continue;
            }
            Set<String> partitionSet = idealState.getPartitionSet();
            String resourceName = idealState.getResourceName();
            if (!resourceMap.containsKey(resourceName)) {
                Resource resource = new Resource(resourceName, cache.getClusterConfig(), cache.getResourceConfig(resourceName));
                resourceMap.put(resourceName, resource);
                if (!idealState.isValid() && !cache.isTaskCache() || idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache()) {
                    resourceToRebalance.put(resourceName, resource);
                }
                resource.setStateModelDefRef(idealState.getStateModelDefRef());
                resource.setStateModelFactoryName(idealState.getStateModelFactoryName());
                resource.setBucketSize(idealState.getBucketSize());
                boolean batchMessageMode = idealState.getBatchMessageMode();
                ClusterConfig clusterConfig = cache.getClusterConfig();
                if (clusterConfig != null) {
                    batchMessageMode |= clusterConfig.getBatchMessageMode();
                }
                resource.setBatchMessageMode(batchMessageMode);
                resource.setResourceGroupName(idealState.getResourceGroupName());
                resource.setResourceTag(idealState.getInstanceGroupTag());
            }
            for (String partition : partitionSet) {
                addPartition(partition, resourceName, resourceMap);
            }
        }
    }
    // It's important to get partitions from CurrentState as well since the
    // idealState might be removed.
    Map<String, LiveInstance> availableInstances = cache.getLiveInstances();
    if (availableInstances != null && availableInstances.size() > 0) {
        for (LiveInstance instance : availableInstances.values()) {
            String instanceName = instance.getInstanceName();
            String clientSessionId = instance.getSessionId();
            Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, clientSessionId);
            if (currentStateMap == null || currentStateMap.size() == 0) {
                continue;
            }
            for (CurrentState currentState : currentStateMap.values()) {
                String resourceName = currentState.getResourceName();
                Map<String, String> resourceStateMap = currentState.getPartitionStateMap();
                if (resourceStateMap.keySet().isEmpty()) {
                    // don't include empty current state for dropped resource
                    continue;
                }
                // don't overwrite ideal state settings
                if (!resourceMap.containsKey(resourceName)) {
                    addResource(resourceName, resourceMap);
                    Resource resource = resourceMap.get(resourceName);
                    resource.setStateModelDefRef(currentState.getStateModelDefRef());
                    resource.setStateModelFactoryName(currentState.getStateModelFactoryName());
                    resource.setBucketSize(currentState.getBucketSize());
                    resource.setBatchMessageMode(currentState.getBatchMessageMode());
                    if (resource.getStateModelDefRef() == null && !cache.isTaskCache() || resource.getStateModelDefRef() != null && (resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache())) {
                        resourceToRebalance.put(resourceName, resource);
                    }
                    IdealState idealState = idealStates.get(resourceName);
                    if (idealState != null) {
                        resource.setResourceGroupName(idealState.getResourceGroupName());
                        resource.setResourceTag(idealState.getInstanceGroupTag());
                    }
                }
                if (currentState.getStateModelDefRef() == null) {
                    LOG.error("state model def is null." + "resource:" + currentState.getResourceName() + ", partitions: " + currentState.getPartitionStateMap().keySet() + ", states: " + currentState.getPartitionStateMap().values());
                    throw new StageException("State model def is null for resource:" + currentState.getResourceName());
                }
                for (String partition : resourceStateMap.keySet()) {
                    addPartition(partition, resourceName, resourceMap);
                }
            }
        }
    }
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceToRebalance);
}