Examples with Partition org.apache.helix.model.Partition used on opensource projects

Example 1 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class MessageGenerationPhase method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    Map<String, List<Message>> pendingMessagesToCleanUp = new HashMap<>();
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    IntermediateStateOutput intermediateStateOutput = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    if (manager == null || cache == null || resourceMap == null || currentStateOutput == null || intermediateStateOutput == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|DataCache|RESOURCES|CURRENT_STATE|INTERMEDIATE_STATE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    Map<String, String> sessionIdMap = new HashMap<String, String>();
    for (LiveInstance liveInstance : liveInstances.values()) {
        sessionIdMap.put(liveInstance.getInstanceName(), liveInstance.getSessionId());
    }
    MessageGenerationOutput output = new MessageGenerationOutput();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        StateModelDefinition stateModelDef = cache.getStateModelDef(resource.getStateModelDefRef());
        if (stateModelDef == null) {
            logger.error("State Model Definition null, skip generating messages for resource: " + resourceName);
            continue;
        }
        for (Partition partition : resource.getPartitions()) {
            Map<String, String> instanceStateMap = new HashMap<String, String>(intermediateStateOutput.getInstanceStateMap(resourceName, partition));
            Map<String, String> pendingStateMap = currentStateOutput.getPendingStateMap(resourceName, partition);
            for (String instance : pendingStateMap.keySet()) {
                if (!instanceStateMap.containsKey(instance)) {
                    instanceStateMap.put(instance, NO_DESIRED_STATE);
                }
            }
            // we should generate message based on the desired-state priority
            // so keep generated messages in a temp map keyed by state
            // desired-state->list of generated-messages
            Map<String, List<Message>> messageMap = new HashMap<String, List<Message>>();
            for (String instanceName : instanceStateMap.keySet()) {
                String desiredState = instanceStateMap.get(instanceName);
                String currentState = currentStateOutput.getCurrentState(resourceName, partition, instanceName);
                if (currentState == null) {
                    currentState = stateModelDef.getInitialState();
                }
                Message pendingMessage = currentStateOutput.getPendingState(resourceName, partition, instanceName);
                boolean isCancellationEnabled = cache.getClusterConfig().isStateTransitionCancelEnabled();
                Message cancellationMessage = currentStateOutput.getCancellationState(resourceName, partition, instanceName);
                String nextState = stateModelDef.getNextStateForTransition(currentState, desiredState);
                Message message = null;
                if (shouldCleanUpPendingMessage(pendingMessage, currentState, currentStateOutput.getEndTime(resourceName, partition, instanceName))) {
                    logger.info("Adding pending message {} on instance {} to clean up. Msg: {}->{}, current state of resource {}:{} is {}", pendingMessage.getMsgId(), instanceName, pendingMessage.getFromState(), pendingMessage.getToState(), resourceName, partition, currentState);
                    if (!pendingMessagesToCleanUp.containsKey(instanceName)) {
                        pendingMessagesToCleanUp.put(instanceName, new ArrayList<Message>());
                    }
                    pendingMessagesToCleanUp.get(instanceName).add(pendingMessage);
                }
                if (desiredState.equals(NO_DESIRED_STATE) || desiredState.equalsIgnoreCase(currentState)) {
                    if (desiredState.equals(NO_DESIRED_STATE) || pendingMessage != null && !currentState.equalsIgnoreCase(pendingMessage.getToState())) {
                        message = createStateTransitionCancellationMessage(manager, resource, partition.getPartitionName(), instanceName, sessionIdMap.get(instanceName), stateModelDef.getId(), pendingMessage.getFromState(), pendingMessage.getToState(), null, cancellationMessage, isCancellationEnabled, currentState);
                    }
                } else {
                    if (nextState == null) {
                        logger.error("Unable to find a next state for resource: " + resource.getResourceName() + " partition: " + partition.getPartitionName() + " from stateModelDefinition" + stateModelDef.getClass() + " from:" + currentState + " to:" + desiredState);
                        continue;
                    }
                    if (pendingMessage != null) {
                        String pendingState = pendingMessage.getToState();
                        if (nextState.equalsIgnoreCase(pendingState)) {
                            logger.debug("Message already exists for " + instanceName + " to transit " + resource.getResourceName() + "." + partition.getPartitionName() + " from " + currentState + " to " + nextState);
                        } else if (currentState.equalsIgnoreCase(pendingState)) {
                            logger.info("Message hasn't been removed for " + instanceName + " to transit " + resource.getResourceName() + "." + partition.getPartitionName() + " to " + pendingState + ", desiredState: " + desiredState);
                        } else {
                            logger.info("IdealState changed before state transition completes for " + resource.getResourceName() + "." + partition.getPartitionName() + " on " + instanceName + ", pendingState: " + pendingState + ", currentState: " + currentState + ", nextState: " + nextState);
                            message = createStateTransitionCancellationMessage(manager, resource, partition.getPartitionName(), instanceName, sessionIdMap.get(instanceName), stateModelDef.getId(), pendingMessage.getFromState(), pendingState, nextState, cancellationMessage, isCancellationEnabled, currentState);
                        }
                    } else {
                        // Create new state transition message
                        message = createStateTransitionMessage(manager, resource, partition.getPartitionName(), instanceName, currentState, nextState, sessionIdMap.get(instanceName), stateModelDef.getId());
                    }
                }
                if (message != null) {
                    IdealState idealState = cache.getIdealState(resourceName);
                    if (idealState != null && idealState.getStateModelDefRef().equalsIgnoreCase(DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE)) {
                        if (idealState.getRecord().getMapField(partition.getPartitionName()) != null) {
                            message.getRecord().setMapField(Message.Attributes.INNER_MESSAGE.toString(), idealState.getRecord().getMapField(partition.getPartitionName()));
                        }
                    }
                    int timeout = getTimeOut(cache.getClusterConfig(), cache.getResourceConfig(resourceName), currentState, nextState, idealState, partition);
                    if (timeout > 0) {
                        message.setExecutionTimeout(timeout);
                    }
                    message.setAttribute(Message.Attributes.ClusterEventName, event.getEventType().name());
                    // output.addMessage(resourceName, partition, message);
                    if (!messageMap.containsKey(desiredState)) {
                        messageMap.put(desiredState, new ArrayList<Message>());
                    }
                    messageMap.get(desiredState).add(message);
                }
            }
            // add generated messages to output according to state priority
            List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
            for (String state : statesPriorityList) {
                if (messageMap.containsKey(state)) {
                    for (Message message : messageMap.get(state)) {
                        output.addMessage(resourceName, partition, message);
                    }
                }
            }
        }
    // end of for-each-partition
    }
    // Asynchronously clean up pending messages if necessary
    if (!pendingMessagesToCleanUp.isEmpty()) {
        schedulePendingMessageCleanUp(pendingMessagesToCleanUp, cache.getAsyncTasksThreadPool(), manager.getHelixDataAccessor());
    }
    event.addAttribute(AttributeName.MESSAGES_ALL.name(), output);
}

Example 2 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class MessageSelectionStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    MessageGenerationOutput messageGenOutput = event.getAttribute(AttributeName.MESSAGES_ALL.name());
    if (cache == null || resourceMap == null || currentStateOutput == null || messageGenOutput == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires DataCache|RESOURCES|CURRENT_STATE|MESSAGES_ALL");
    }
    MessageSelectionStageOutput output = new MessageSelectionStageOutput();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        StateModelDefinition stateModelDef = cache.getStateModelDef(resource.getStateModelDefRef());
        Map<String, Integer> stateTransitionPriorities = getStateTransitionPriorityMap(stateModelDef);
        IdealState idealState = cache.getIdealState(resourceName);
        Map<String, Bounds> stateConstraints = computeStateConstraints(stateModelDef, idealState, cache);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageGenOutput.getMessages(resourceName, partition);
            List<Message> selectedMessages = selectMessages(cache.getLiveInstances(), currentStateOutput.getCurrentStateMap(resourceName, partition), currentStateOutput.getPendingMessageMap(resourceName, partition), messages, stateConstraints, stateTransitionPriorities, stateModelDef, resource.isP2PMessageEnabled());
            output.addMessages(resourceName, partition, selectedMessages);
        }
    }
    event.addAttribute(AttributeName.MESSAGES_SELECTED.name(), output);
}

Example 3 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class MessageThrottleStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    MessageSelectionStageOutput msgSelectionOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name());
    if (cache == null || resourceMap == null || msgSelectionOutput == null) {
        throw new StageException("Missing attributes in event: " + event + ". Requires ClusterDataCache|RESOURCES|MESSAGES_SELECTED");
    }
    MessageThrottleStageOutput output = new MessageThrottleStageOutput();
    ClusterConstraints constraint = cache.getConstraint(ConstraintType.MESSAGE_CONSTRAINT);
    Map<String, Integer> throttleCounterMap = new HashMap<String, Integer>();
    if (constraint != null) {
        // go through all pending messages, they should be counted but not throttled
        for (String instance : cache.getLiveInstances().keySet()) {
            throttle(throttleCounterMap, constraint, new ArrayList<Message>(cache.getMessages(instance).values()), false);
        }
    }
    // assume messages should be sorted by state transition priority in messageSelection stage
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = msgSelectionOutput.getMessages(resourceName, partition);
            if (constraint != null && messages != null && messages.size() > 0) {
                messages = throttle(throttleCounterMap, constraint, messages, true);
            }
            output.addMessages(resourceName, partition, messages);
        }
    }
    event.addAttribute(AttributeName.MESSAGES_THROTTLE.name(), output);
}

Example 4 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class PersistAssignmentStage method hasInstanceMapChanged.

private boolean hasInstanceMapChanged(Map<Partition, Map<String, String>> newAssiments, IdealState idealState) {
    Set<Partition> partitions = new HashSet<Partition>(newAssiments.keySet());
    for (String p : idealState.getPartitionSet()) {
        partitions.add(new Partition(p));
    }
    for (Partition partition : partitions) {
        Map<String, String> instanceMap = newAssiments.get(partition);
        Map<String, String> existInstanceMap = idealState.getInstanceStateMap(partition.getPartitionName());
        if (instanceMap == null && existInstanceMap == null) {
            continue;
        }
        if (instanceMap == null || existInstanceMap == null || !instanceMap.equals(existInstanceMap)) {
            return true;
        }
    }
    return false;
}

Example 5 with Partition

use of org.apache.helix.model.Partition in project helix by apache.

the class PersistAssignmentStage method convertAssignmentPersisted.

/**
 * TODO: This is a temporary hacky for back-compatible support of Espresso and Databus, we should
 * get rid of this conversion as soon as possible. --- Lei, 2016/9/9.
 */
private Map<Partition, Map<String, String>> convertAssignmentPersisted(Resource resource, IdealState idealState, Map<Partition, Map<String, String>> assignments) {
    String stateModelDef = idealState.getStateModelDefRef();
    /**
     * Only convert for MasterSlave resources
     */
    if (!stateModelDef.equals(BuiltInStateModelDefinitions.MasterSlave.name()) || idealState.getRebalanceMode().equals(IdealState.RebalanceMode.FULL_AUTO)) {
        return assignments;
    }
    Map<Partition, Map<String, String>> assignmentToPersist = new HashMap<Partition, Map<String, String>>();
    for (Partition partition : resource.getPartitions()) {
        Map<String, String> instanceMap = new HashMap<String, String>();
        Map<String, String> assignment = assignments.get(partition);
        if (assignment != null) {
            instanceMap.putAll(assignment);
        }
        List<String> preferenceList = idealState.getPreferenceList(partition.getPartitionName());
        if (preferenceList == null) {
            preferenceList = Collections.emptyList();
        }
        Set<String> nodeList = new HashSet<String>(preferenceList);
        nodeList.addAll(assignment.keySet());
        boolean hasMaster = false;
        for (String ins : nodeList) {
            String state = instanceMap.get(ins);
            if (state == null || (!state.equals(MasterSlaveSMD.States.SLAVE.name()) && !state.equals(MasterSlaveSMD.States.MASTER.name()))) {
                instanceMap.put(ins, MasterSlaveSMD.States.SLAVE.name());
            }
            if (state != null && state.equals(MasterSlaveSMD.States.MASTER.name())) {
                hasMaster = true;
            }
        }
        // if no master, just pick the first node in the preference list as the master.
        if (!hasMaster && preferenceList.size() > 0) {
            instanceMap.put(preferenceList.get(0), MasterSlaveSMD.States.MASTER.name());
        }
        assignmentToPersist.put(partition, instanceMap);
    }
    return assignmentToPersist;
}