Examples with ClusterStatusMonitor org.apache.helix.monitoring.mbeans.ClusterStatusMonitor used on opensource projects

Example 6 with ClusterStatusMonitor

use of org.apache.helix.monitoring.mbeans.ClusterStatusMonitor in project helix by apache.

the class CurrentStateComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (cache == null || resourceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (LiveInstance instance : liveInstances.values()) {
        String instanceName = instance.getInstanceName();
        String instanceSessionId = instance.getSessionId();
        // update pending messages
        Map<String, Message> messages = cache.getMessages(instanceName);
        updatePendingMessages(instance, messages.values(), currentStateOutput, resourceMap);
        // update current states.
        Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, instanceSessionId);
        updateCurrentStates(instance, currentStateMap.values(), currentStateOutput, resourceMap);
    }
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        updateMissingTopStateStatus(cache, clusterStatusMonitor, resourceMap, currentStateOutput);
    }
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
}

Example 7 with ClusterStatusMonitor

use of org.apache.helix.monitoring.mbeans.ClusterStatusMonitor in project helix by apache.

the class IntermediateStateCalcStage method compute.

private IntermediateStateOutput compute(ClusterEvent event, Map<String, Resource> resourceMap, CurrentStateOutput currentStateOutput, BestPossibleStateOutput bestPossibleStateOutput) {
    // for each resource
    // get the best possible state and current state
    // try to bring immediate state close to best possible state until
    // the possible pending state transition numbers reach the set throttle number.
    IntermediateStateOutput output = new IntermediateStateOutput();
    ClusterDataCache dataCache = event.getAttribute(AttributeName.ClusterDataCache.name());
    StateTransitionThrottleController throttleController = new StateTransitionThrottleController(resourceMap.keySet(), dataCache.getClusterConfig(), dataCache.getLiveInstances().keySet());
    // Resource level prioritization with numerical sortable field.
    // If no value has been set, it will be treated as lowest priority.
    List<ResourcePriority> prioritizedResourceList = new ArrayList<ResourcePriority>();
    for (String resourceName : resourceMap.keySet()) {
        prioritizedResourceList.add(new ResourcePriority(resourceName, Integer.MIN_VALUE));
    }
    // Not have resource level prioritization if user did not set the field name
    if (dataCache.getClusterConfig().getResourcePriorityField() != null) {
        String priorityField = dataCache.getClusterConfig().getResourcePriorityField();
        for (ResourcePriority resourcePriority : prioritizedResourceList) {
            String resourceName = resourcePriority.getResourceName();
            // Try to fetch it from ideal state. Otherwise will treated as lowest priority
            if (dataCache.getResourceConfig(resourceName) != null && dataCache.getResourceConfig(resourceName).getSimpleConfig(priorityField) != null) {
                resourcePriority.setPriority(dataCache.getResourceConfig(resourceName).getSimpleConfig(priorityField));
            } else if (dataCache.getIdealState(resourceName) != null && dataCache.getIdealState(resourceName).getRecord().getSimpleField(priorityField) != null) {
                resourcePriority.setPriority(dataCache.getIdealState(resourceName).getRecord().getSimpleField(priorityField));
            }
        }
        Collections.sort(prioritizedResourceList, new ResourcePriortiyComparator());
    }
    // Update cluster status monitor mbean
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    for (ResourcePriority resourcePriority : prioritizedResourceList) {
        String resourceName = resourcePriority.getResourceName();
        Resource resource = resourceMap.get(resourceName);
        IdealState idealState = dataCache.getIdealState(resourceName);
        if (idealState == null) {
            // if ideal state is deleted, use an empty one
            logger.info("resource:" + resourceName + " does not exist anymore");
            idealState = new IdealState(resourceName);
            idealState.setStateModelDefRef(resource.getStateModelDefRef());
        }
        PartitionStateMap intermediatePartitionStateMap = computeIntermediatePartitionState(dataCache, clusterStatusMonitor, idealState, resourceMap.get(resourceName), currentStateOutput, bestPossibleStateOutput.getPartitionStateMap(resourceName), bestPossibleStateOutput.getPreferenceLists(resourceName), throttleController);
        output.setState(resourceName, intermediatePartitionStateMap);
    }
    return output;
}

Example 8 with ClusterStatusMonitor

use of org.apache.helix.monitoring.mbeans.ClusterStatusMonitor in project helix by apache.

the class TaskAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    MessageThrottleStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    if (manager == null || resourceMap == null || messageOutput == null || cache == null || liveInstanceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageOutput.getMessages(resourceName, partition);
            messagesToSend.addAll(messages);
        }
    }
    List<Message> outputMessages = batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap, manager.getProperties());
    sendMessages(dataAccessor, outputMessages);
    // TODO: Need also count messages from task rebalancer
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.increaseMessageReceived(outputMessages);
        }
    }
    long cacheStart = System.currentTimeMillis();
    cache.cacheMessages(outputMessages);
    long cacheEnd = System.currentTimeMillis();
    logger.debug("Caching messages took " + (cacheEnd - cacheStart) + " ms");
}