Examples with Resource org.apache.helix.model.Resource used on opensource projects

Example 26 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class CurrentStateComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (cache == null || resourceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache|RESOURCE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
    for (LiveInstance instance : liveInstances.values()) {
        String instanceName = instance.getInstanceName();
        String instanceSessionId = instance.getSessionId();
        // update pending messages
        Map<String, Message> messages = cache.getMessages(instanceName);
        updatePendingMessages(instance, messages.values(), currentStateOutput, resourceMap);
        // update current states.
        Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, instanceSessionId);
        updateCurrentStates(instance, currentStateMap.values(), currentStateOutput, resourceMap);
    }
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        updateMissingTopStateStatus(cache, clusterStatusMonitor, resourceMap, currentStateOutput);
    }
    event.addAttribute(AttributeName.CURRENT_STATE.name(), currentStateOutput);
}

Example 27 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class PersistAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    ClusterConfig clusterConfig = cache.getClusterConfig();
    if (!clusterConfig.isPersistBestPossibleAssignment() && !clusterConfig.isPersistIntermediateAssignment()) {
        return;
    }
    BestPossibleStateOutput bestPossibleAssignment = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    HelixManager helixManager = event.getAttribute(AttributeName.helixmanager.name());
    HelixDataAccessor accessor = helixManager.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    for (String resourceId : bestPossibleAssignment.resourceSet()) {
        Resource resource = resourceMap.get(resourceId);
        if (resource != null) {
            final IdealState idealState = cache.getIdealState(resourceId);
            if (idealState == null) {
                LOG.warn("IdealState not found for resource " + resourceId);
                continue;
            }
            IdealState.RebalanceMode mode = idealState.getRebalanceMode();
            if (!mode.equals(IdealState.RebalanceMode.SEMI_AUTO) && !mode.equals(IdealState.RebalanceMode.FULL_AUTO)) {
                // do not persist assignment for resource in neither semi or full auto.
                continue;
            }
            boolean needPersist = false;
            if (mode.equals(IdealState.RebalanceMode.FULL_AUTO)) {
                // persist preference list in ful-auto mode.
                Map<String, List<String>> newLists = bestPossibleAssignment.getPreferenceLists(resourceId);
                if (newLists != null && hasPreferenceListChanged(newLists, idealState)) {
                    idealState.setPreferenceLists(newLists);
                    needPersist = true;
                }
            }
            PartitionStateMap partitionStateMap = bestPossibleAssignment.getPartitionStateMap(resourceId);
            if (clusterConfig.isPersistIntermediateAssignment()) {
                IntermediateStateOutput intermediateAssignment = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
                partitionStateMap = intermediateAssignment.getPartitionStateMap(resourceId);
            }
            // TODO: temporary solution for Espresso/Dbus backcompatible, should remove this.
            Map<Partition, Map<String, String>> assignmentToPersist = convertAssignmentPersisted(resource, idealState, partitionStateMap.getStateMap());
            if (assignmentToPersist != null && hasInstanceMapChanged(assignmentToPersist, idealState)) {
                for (Partition partition : assignmentToPersist.keySet()) {
                    Map<String, String> instanceMap = assignmentToPersist.get(partition);
                    idealState.setInstanceStateMap(partition.getPartitionName(), instanceMap);
                }
                needPersist = true;
            }
            if (needPersist) {
                // Update instead of set to ensure any intermediate changes that the controller does not update are kept.
                accessor.updateProperty(keyBuilder.idealStates(resourceId), new DataUpdater<ZNRecord>() {

                    @Override
                    public ZNRecord update(ZNRecord current) {
                        if (current != null) {
                            // Overwrite MapFields and ListFields items with the same key.
                            // Note that default merge will keep old values in the maps or lists unchanged, which is not desired.
                            current.getMapFields().clear();
                            current.getMapFields().putAll(idealState.getRecord().getMapFields());
                            current.getListFields().putAll(idealState.getRecord().getListFields());
                        }
                        return current;
                    }
                }, idealState);
            }
        }
    }
}

Example 28 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class ResourceComputationStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache");
    }
    Map<String, IdealState> idealStates = cache.getIdealStates();
    Map<String, Resource> resourceMap = new LinkedHashMap<String, Resource>();
    Map<String, Resource> resourceToRebalance = new LinkedHashMap<>();
    if (idealStates != null && idealStates.size() > 0) {
        for (IdealState idealState : idealStates.values()) {
            if (idealState == null) {
                continue;
            }
            Set<String> partitionSet = idealState.getPartitionSet();
            String resourceName = idealState.getResourceName();
            if (!resourceMap.containsKey(resourceName)) {
                Resource resource = new Resource(resourceName, cache.getClusterConfig(), cache.getResourceConfig(resourceName));
                resourceMap.put(resourceName, resource);
                if (!idealState.isValid() && !cache.isTaskCache() || idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !idealState.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache()) {
                    resourceToRebalance.put(resourceName, resource);
                }
                resource.setStateModelDefRef(idealState.getStateModelDefRef());
                resource.setStateModelFactoryName(idealState.getStateModelFactoryName());
                resource.setBucketSize(idealState.getBucketSize());
                boolean batchMessageMode = idealState.getBatchMessageMode();
                ClusterConfig clusterConfig = cache.getClusterConfig();
                if (clusterConfig != null) {
                    batchMessageMode |= clusterConfig.getBatchMessageMode();
                }
                resource.setBatchMessageMode(batchMessageMode);
                resource.setResourceGroupName(idealState.getResourceGroupName());
                resource.setResourceTag(idealState.getInstanceGroupTag());
            }
            for (String partition : partitionSet) {
                addPartition(partition, resourceName, resourceMap);
            }
        }
    }
    // It's important to get partitions from CurrentState as well since the
    // idealState might be removed.
    Map<String, LiveInstance> availableInstances = cache.getLiveInstances();
    if (availableInstances != null && availableInstances.size() > 0) {
        for (LiveInstance instance : availableInstances.values()) {
            String instanceName = instance.getInstanceName();
            String clientSessionId = instance.getSessionId();
            Map<String, CurrentState> currentStateMap = cache.getCurrentState(instanceName, clientSessionId);
            if (currentStateMap == null || currentStateMap.size() == 0) {
                continue;
            }
            for (CurrentState currentState : currentStateMap.values()) {
                String resourceName = currentState.getResourceName();
                Map<String, String> resourceStateMap = currentState.getPartitionStateMap();
                if (resourceStateMap.keySet().isEmpty()) {
                    // don't include empty current state for dropped resource
                    continue;
                }
                // don't overwrite ideal state settings
                if (!resourceMap.containsKey(resourceName)) {
                    addResource(resourceName, resourceMap);
                    Resource resource = resourceMap.get(resourceName);
                    resource.setStateModelDefRef(currentState.getStateModelDefRef());
                    resource.setStateModelFactoryName(currentState.getStateModelFactoryName());
                    resource.setBucketSize(currentState.getBucketSize());
                    resource.setBatchMessageMode(currentState.getBatchMessageMode());
                    if (resource.getStateModelDefRef() == null && !cache.isTaskCache() || resource.getStateModelDefRef() != null && (resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && cache.isTaskCache() || !resource.getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME) && !cache.isTaskCache())) {
                        resourceToRebalance.put(resourceName, resource);
                    }
                    IdealState idealState = idealStates.get(resourceName);
                    if (idealState != null) {
                        resource.setResourceGroupName(idealState.getResourceGroupName());
                        resource.setResourceTag(idealState.getInstanceGroupTag());
                    }
                }
                if (currentState.getStateModelDefRef() == null) {
                    LOG.error("state model def is null." + "resource:" + currentState.getResourceName() + ", partitions: " + currentState.getPartitionStateMap().keySet() + ", states: " + currentState.getPartitionStateMap().values());
                    throw new StageException("State model def is null for resource:" + currentState.getResourceName());
                }
                for (String partition : resourceStateMap.keySet()) {
                    addPartition(partition, resourceName, resourceMap);
                }
            }
        }
    }
    event.addAttribute(AttributeName.RESOURCES.name(), resourceMap);
    event.addAttribute(AttributeName.RESOURCES_TO_REBALANCE.name(), resourceToRebalance);
}

Example 29 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class ResourceComputationStage method addPartition.

private void addPartition(String partition, String resourceName, Map<String, Resource> resourceMap) {
    if (resourceName == null || partition == null || resourceMap == null) {
        return;
    }
    if (!resourceMap.containsKey(resourceName)) {
        resourceMap.put(resourceName, new Resource(resourceName));
    }
    Resource resource = resourceMap.get(resourceName);
    resource.addPartition(partition);
}

Example 30 with Resource

use of org.apache.helix.model.Resource in project helix by apache.

the class TargetExteralViewCalcStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    ClusterConfig clusterConfig = cache.getClusterConfig();
    if (cache.isTaskCache() || !clusterConfig.isTargetExternalViewEnabled()) {
        return;
    }
    HelixManager helixManager = event.getAttribute(AttributeName.helixmanager.name());
    HelixDataAccessor accessor = helixManager.getHelixDataAccessor();
    if (!accessor.getBaseDataAccessor().exists(accessor.keyBuilder().targetExternalViews().getPath(), AccessOption.PERSISTENT)) {
        accessor.getBaseDataAccessor().create(accessor.keyBuilder().targetExternalViews().getPath(), null, AccessOption.PERSISTENT);
    }
    BestPossibleStateOutput bestPossibleAssignments = event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name());
    IntermediateStateOutput intermediateAssignments = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    List<PropertyKey> keys = new ArrayList<>();
    List<ExternalView> targetExternalViews = new ArrayList<>();
    for (String resourceName : bestPossibleAssignments.resourceSet()) {
        if (cache.getIdealState(resourceName) == null || cache.getIdealState(resourceName).isExternalViewDisabled()) {
            continue;
        }
        Resource resource = resourceMap.get(resourceName);
        if (resource != null) {
            PartitionStateMap partitionStateMap = intermediateAssignments.getPartitionStateMap(resourceName);
            Map<String, Map<String, String>> intermediateAssignment = convertToMapFields(partitionStateMap.getStateMap());
            Map<String, List<String>> preferenceLists = bestPossibleAssignments.getPreferenceLists(resourceName);
            boolean needPersist = false;
            ExternalView targetExternalView = cache.getTargetExternalView(resourceName);
            if (targetExternalView == null) {
                targetExternalView = new ExternalView(resourceName);
                targetExternalView.getRecord().getSimpleFields().putAll(cache.getIdealState(resourceName).getRecord().getSimpleFields());
                needPersist = true;
            }
            if (preferenceLists != null && !targetExternalView.getRecord().getListFields().equals(preferenceLists)) {
                targetExternalView.getRecord().setListFields(preferenceLists);
                needPersist = true;
            }
            if (intermediateAssignment != null && !targetExternalView.getRecord().getMapFields().equals(intermediateAssignment)) {
                targetExternalView.getRecord().setMapFields(intermediateAssignment);
                needPersist = true;
            }
            if (needPersist) {
                keys.add(accessor.keyBuilder().targetExternalView(resourceName));
                targetExternalViews.add(targetExternalView);
                cache.updateTargetExternalView(resourceName, targetExternalView);
            }
        }
    }
    accessor.setChildren(keys, targetExternalViews);
}