Examples with PartitionStateMap org.apache.helix.controller.common.PartitionStateMap used on opensource projects

Example 6 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class TestStateTransitionPrirority method updateCurrentStateForPartitionLevelPriority.

private void updateCurrentStateForPartitionLevelPriority(List<String> partitionPriority, CurrentStateOutput currentStateOutput, String resourceName, Map<String, Map<String, String>> bestPossibleMap) {
    IntermediateStateOutput output = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    PartitionStateMap partitionStateMap = output.getPartitionStateMap(resourceName);
    for (Partition partition : partitionStateMap.getStateMap().keySet()) {
        Map<String, String> instanceStateMap = bestPossibleMap.get(partition.getPartitionName());
        if (partitionStateMap.getPartitionMap(partition).equals(instanceStateMap) && !partitionPriority.contains(partition.getPartitionName())) {
            partitionPriority.add(partition.getPartitionName());
            for (String instanceName : instanceStateMap.keySet()) {
                currentStateOutput.setCurrentState(resourceName, partition, instanceName, instanceStateMap.get(instanceName));
            }
            break;
        }
    }
}

Example 7 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class IntermediateStateCalcStage method compute.

private IntermediateStateOutput compute(ClusterEvent event, Map<String, Resource> resourceMap, CurrentStateOutput currentStateOutput, BestPossibleStateOutput bestPossibleStateOutput) {
    // for each resource
    // get the best possible state and current state
    // try to bring immediate state close to best possible state until
    // the possible pending state transition numbers reach the set throttle number.
    IntermediateStateOutput output = new IntermediateStateOutput();
    ClusterDataCache dataCache = event.getAttribute(AttributeName.ClusterDataCache.name());
    StateTransitionThrottleController throttleController = new StateTransitionThrottleController(resourceMap.keySet(), dataCache.getClusterConfig(), dataCache.getLiveInstances().keySet());
    // Resource level prioritization with numerical sortable field.
    // If no value has been set, it will be treated as lowest priority.
    List<ResourcePriority> prioritizedResourceList = new ArrayList<ResourcePriority>();
    for (String resourceName : resourceMap.keySet()) {
        prioritizedResourceList.add(new ResourcePriority(resourceName, Integer.MIN_VALUE));
    }
    // Not have resource level prioritization if user did not set the field name
    if (dataCache.getClusterConfig().getResourcePriorityField() != null) {
        String priorityField = dataCache.getClusterConfig().getResourcePriorityField();
        for (ResourcePriority resourcePriority : prioritizedResourceList) {
            String resourceName = resourcePriority.getResourceName();
            // Try to fetch it from ideal state. Otherwise will treated as lowest priority
            if (dataCache.getResourceConfig(resourceName) != null && dataCache.getResourceConfig(resourceName).getSimpleConfig(priorityField) != null) {
                resourcePriority.setPriority(dataCache.getResourceConfig(resourceName).getSimpleConfig(priorityField));
            } else if (dataCache.getIdealState(resourceName) != null && dataCache.getIdealState(resourceName).getRecord().getSimpleField(priorityField) != null) {
                resourcePriority.setPriority(dataCache.getIdealState(resourceName).getRecord().getSimpleField(priorityField));
            }
        }
        Collections.sort(prioritizedResourceList, new ResourcePriortiyComparator());
    }
    // Update cluster status monitor mbean
    ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
    for (ResourcePriority resourcePriority : prioritizedResourceList) {
        String resourceName = resourcePriority.getResourceName();
        Resource resource = resourceMap.get(resourceName);
        IdealState idealState = dataCache.getIdealState(resourceName);
        if (idealState == null) {
            // if ideal state is deleted, use an empty one
            logger.info("resource:" + resourceName + " does not exist anymore");
            idealState = new IdealState(resourceName);
            idealState.setStateModelDefRef(resource.getStateModelDefRef());
        }
        PartitionStateMap intermediatePartitionStateMap = computeIntermediatePartitionState(dataCache, clusterStatusMonitor, idealState, resourceMap.get(resourceName), currentStateOutput, bestPossibleStateOutput.getPartitionStateMap(resourceName), bestPossibleStateOutput.getPreferenceLists(resourceName), throttleController);
        output.setState(resourceName, intermediatePartitionStateMap);
    }
    return output;
}

Example 8 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class IntermediateStateCalcStage method validateMaxPartitionsPerInstance.

private void validateMaxPartitionsPerInstance(ClusterEvent event, ClusterDataCache cache, IntermediateStateOutput intermediateStateOutput, int maxPartitionPerInstance) {
    Map<String, PartitionStateMap> resourceStatesMap = intermediateStateOutput.getResourceStatesMap();
    Map<String, Integer> instancePartitionCounts = new HashMap<>();
    for (String resource : resourceStatesMap.keySet()) {
        IdealState idealState = cache.getIdealState(resource);
        if (idealState != null && idealState.getStateModelDefRef().equals(BuiltInStateModelDefinitions.Task.name())) {
            // ignore task here. Task has its own throttling logic
            continue;
        }
        PartitionStateMap partitionStateMap = resourceStatesMap.get(resource);
        Map<Partition, Map<String, String>> stateMaps = partitionStateMap.getStateMap();
        for (Partition p : stateMaps.keySet()) {
            Map<String, String> stateMap = stateMaps.get(p);
            for (String instance : stateMap.keySet()) {
                // ignore replica to be dropped.
                String state = stateMap.get(instance);
                if (state.equals(HelixDefinedState.DROPPED.name())) {
                    continue;
                }
                if (!instancePartitionCounts.containsKey(instance)) {
                    instancePartitionCounts.put(instance, 0);
                }
                int partitionCount = instancePartitionCounts.get(instance);
                partitionCount++;
                if (partitionCount > maxPartitionPerInstance) {
                    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
                    String errMsg = String.format("Partition count to be assigned to instance %s is greater than %d. Stop rebalance and pause the cluster %s", instance, maxPartitionPerInstance, cache.getClusterName());
                    if (manager != null) {
                        manager.getClusterManagmentTool().enableMaintenanceMode(manager.getClusterName(), true, errMsg);
                    } else {
                        logger.error("Failed to pause cluster, HelixManager is not set!");
                    }
                    throw new HelixException(errMsg);
                }
                instancePartitionCounts.put(instance, partitionCount);
            }
        }
    }
}

Example 9 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class PersistAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    ClusterConfig clusterConfig = cache.getClusterConfig();
    if (!clusterConfig.isPersistBestPossibleAssignment() && !clusterConfig.isPersistIntermediateAssignment()) {
        return;
    }
    BestPossibleStateOutput bestPossibleAssignment = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    HelixManager helixManager = event.getAttribute(AttributeName.helixmanager.name());
    HelixDataAccessor accessor = helixManager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    for (String resourceId : bestPossibleAssignment.resourceSet()) {
        Resource resource = resourceMap.get(resourceId);
        if (resource != null) {
            final IdealState idealState = cache.getIdealState(resourceId);
            if (idealState == null) {
                LOG.warn("IdealState not found for resource " + resourceId);
                continue;
            }
            IdealState.RebalanceMode mode = idealState.getRebalanceMode();
            if (!mode.equals(IdealState.RebalanceMode.SEMI_AUTO) && !mode.equals(IdealState.RebalanceMode.FULL_AUTO)) {
                // do not persist assignment for resource in neither semi or full auto.
                continue;
            }
            boolean needPersist = false;
            if (mode.equals(IdealState.RebalanceMode.FULL_AUTO)) {
                // persist preference list in ful-auto mode.
                Map<String, List<String>> newLists = bestPossibleAssignment.getPreferenceLists(resourceId);
                if (newLists != null && hasPreferenceListChanged(newLists, idealState)) {
                    idealState.setPreferenceLists(newLists);
                    needPersist = true;
                }
            }
            PartitionStateMap partitionStateMap = bestPossibleAssignment.getPartitionStateMap(resourceId);
            if (clusterConfig.isPersistIntermediateAssignment()) {
                IntermediateStateOutput intermediateAssignment = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
                partitionStateMap = intermediateAssignment.getPartitionStateMap(resourceId);
            }
            // TODO: temporary solution for Espresso/Dbus backcompatible, should remove this.
            Map<Partition, Map<String, String>> assignmentToPersist = convertAssignmentPersisted(resource, idealState, partitionStateMap.getStateMap());
            if (assignmentToPersist != null && hasInstanceMapChanged(assignmentToPersist, idealState)) {
                for (Partition partition : assignmentToPersist.keySet()) {
                    Map<String, String> instanceMap = assignmentToPersist.get(partition);
                    idealState.setInstanceStateMap(partition.getPartitionName(), instanceMap);
                }
                needPersist = true;
            }
            if (needPersist) {
                // Update instead of set to ensure any intermediate changes that the controller does not update are kept.
                accessor.updateProperty(keyBuilder.idealStates(resourceId), new DataUpdater<ZNRecord>() {

                    @Override
                    public ZNRecord update(ZNRecord current) {
                        if (current != null) {
                            // Overwrite MapFields and ListFields items with the same key.
                            // Note that default merge will keep old values in the maps or lists unchanged, which is not desired.
                            current.getMapFields().clear();
                            current.getMapFields().putAll(idealState.getRecord().getMapFields());
                            current.getListFields().putAll(idealState.getRecord().getListFields());
                        }
                        return current;
                    }
                }, idealState);
            }
        }
    }
}

Example 10 with PartitionStateMap

use of org.apache.helix.controller.common.PartitionStateMap in project helix by apache.

the class TargetExteralViewCalcStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    ClusterConfig clusterConfig = cache.getClusterConfig();
    if (cache.isTaskCache() || !clusterConfig.isTargetExternalViewEnabled()) {
        return;
    }
    HelixManager helixManager = event.getAttribute(AttributeName.helixmanager.name());
    HelixDataAccessor accessor = helixManager.getHelixDataAccessor();
    if (!accessor.getBaseDataAccessor().exists(accessor.keyBuilder().targetExternalViews().getPath(), AccessOption.PERSISTENT)) {
        accessor.getBaseDataAccessor().create(accessor.keyBuilder().targetExternalViews().getPath(), null, AccessOption.PERSISTENT);
    }
    BestPossibleStateOutput bestPossibleAssignments = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    IntermediateStateOutput intermediateAssignments = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    List<PropertyKey> keys = new ArrayList<>();
    List<ExternalView> targetExternalViews = new ArrayList<>();
    for (String resourceName : bestPossibleAssignments.resourceSet()) {
        if (cache.getIdealState(resourceName) == null || cache.getIdealState(resourceName).isExternalViewDisabled()) {
            continue;
        }
        Resource resource = resourceMap.get(resourceName);
        if (resource != null) {
            PartitionStateMap partitionStateMap = intermediateAssignments.getPartitionStateMap(resourceName);
            Map<String, Map<String, String>> intermediateAssignment = convertToMapFields(partitionStateMap.getStateMap());
            Map<String, List<String>> preferenceLists = bestPossibleAssignments.getPreferenceLists(resourceName);
            boolean needPersist = false;
            ExternalView targetExternalView = cache.getTargetExternalView(resourceName);
            if (targetExternalView == null) {
                targetExternalView = new ExternalView(resourceName);
                targetExternalView.getRecord().getSimpleFields().putAll(cache.getIdealState(resourceName).getRecord().getSimpleFields());
                needPersist = true;
            }
            if (preferenceLists != null && !targetExternalView.getRecord().getListFields().equals(preferenceLists)) {
                targetExternalView.getRecord().setListFields(preferenceLists);
                needPersist = true;
            }
            if (intermediateAssignment != null && !targetExternalView.getRecord().getMapFields().equals(intermediateAssignment)) {
                targetExternalView.getRecord().setMapFields(intermediateAssignment);
                needPersist = true;
            }
            if (needPersist) {
                keys.add(accessor.keyBuilder().targetExternalView(resourceName));
                targetExternalViews.add(targetExternalView);
                cache.updateTargetExternalView(resourceName, targetExternalView);
            }
        }
    }
    accessor.setChildren(keys, targetExternalViews);
}