Examples with Message org.apache.helix.model.Message used on opensource projects

Example 96 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class CurrentStateOutput method getPartitionCountWithState.

private Map<String, Integer> getPartitionCountWithState(String resourceStateModel, String state, Map<String, Map<Partition, Map<String, Object>>> stateMap) {
    Map<String, Integer> currentPartitionCount = new HashMap<>();
    for (String resource : stateMap.keySet()) {
        String stateModel = _resourceStateModelMap.get(resource);
        if ((stateModel != null && stateModel.equals(resourceStateModel)) || (stateModel == null && resourceStateModel == null)) {
            for (Partition partition : stateMap.get(resource).keySet()) {
                Map<String, Object> partitionMessage = stateMap.get(resource).get(partition);
                for (Map.Entry<String, Object> participantMap : partitionMessage.entrySet()) {
                    String participant = participantMap.getKey();
                    if (!currentPartitionCount.containsKey(participant)) {
                        currentPartitionCount.put(participant, 0);
                    }
                    String currState = participantMap.getValue().toString();
                    if (participantMap.getValue() instanceof Message) {
                        currState = ((Message) participantMap.getValue()).getToState();
                    }
                    if ((currState != null && currState.equals(state)) || (currState == null && state == null)) {
                        currentPartitionCount.put(participant, currentPartitionCount.get(participant) + 1);
                    }
                }
            }
        }
    }
    return currentPartitionCount;
}

Example 97 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class MessageGenerationPhase method createStateTransitionCancellationMessage.

private Message createStateTransitionCancellationMessage(HelixManager manager, Resource resource, String partitionName, String instanceName, String sessionId, String stateModelDefName, String fromState, String toState, String nextState, Message cancellationMessage, boolean isCancellationEnabled, String currentState) {
    if (isCancellationEnabled && cancellationMessage == null) {
        logger.info("Send cancellation message of the state transition for " + resource.getResourceName() + "." + partitionName + " on " + instanceName + ", currentState: " + currentState + ", nextState: " + (nextState == null ? "N/A" : nextState));
        String uuid = UUID.randomUUID().toString();
        Message message = new Message(MessageType.STATE_TRANSITION_CANCELLATION, uuid);
        message.setSrcName(manager.getInstanceName());
        message.setTgtName(instanceName);
        message.setMsgState(MessageState.NEW);
        message.setPartitionName(partitionName);
        message.setResourceName(resource.getResourceName());
        message.setFromState(fromState);
        message.setToState(toState);
        message.setTgtSessionId(sessionId);
        message.setSrcSessionId(manager.getSessionId());
        message.setStateModelDef(stateModelDefName);
        message.setStateModelFactoryName(resource.getStateModelFactoryname());
        message.setBucketSize(resource.getBucketSize());
        return message;
    }
    return null;
}

Example 98 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class MessageSelectionStage method selectMessages.

// TODO: This method deserves its own class. The class should not understand helix but
// just be able to solve the problem using the algo. I think the method is following that
// but if we don't move it to another class its quite easy to break that contract
/**
 * greedy message selection algorithm: 1) calculate CS+PS state lower/upper-bounds 2)
 * group messages by state transition and sorted by priority 3) from highest priority to
 * lowest, for each message group with the same transition add message one by one and
 * make sure state constraint is not violated update state lower/upper-bounds when a new
 * message is selected.
 *
 * @param liveInstances
 * @param currentStates
 * @param pendingMessages
 * @param messages
 * @param stateConstraints
 * @param stateTransitionPriorities
 * @param stateModelDef
 * @return
 */
List<Message> selectMessages(Map<String, LiveInstance> liveInstances, Map<String, String> currentStates, Map<String, Message> pendingMessages, List<Message> messages, Map<String, Bounds> stateConstraints, final Map<String, Integer> stateTransitionPriorities, StateModelDefinition stateModelDef, boolean p2pMessageEnabled) {
    if (messages == null || messages.isEmpty()) {
        return Collections.emptyList();
    }
    List<Message> selectedMessages = new ArrayList<>();
    Map<String, Integer> stateCnts = new HashMap<>();
    String initialState = stateModelDef.getInitialState();
    // count currentState, if no currentState, count as in initialState
    for (String instance : liveInstances.keySet()) {
        String state = initialState;
        if (currentStates.containsKey(instance)) {
            state = currentStates.get(instance);
        }
        increaseStateCnt(stateConstraints, state, stateCnts);
    }
    // count pendingStates
    for (String instance : pendingMessages.keySet()) {
        Message message = pendingMessages.get(instance);
        increaseStateCnt(stateConstraints, message.getToState(), stateCnts);
        increaseStateCnt(stateConstraints, message.getFromState(), stateCnts);
    }
    // group messages based on state transition priority
    Map<Integer, List<Message>> messagesGroupByStateTransitPriority = new TreeMap<>();
    /* record all state transition messages that transition a replica from top-state */
    List<Message> fromTopStateMessages = new LinkedList<>();
    for (Message message : messages) {
        if (message.getMsgType().equals(Message.MessageType.STATE_TRANSITION_CANCELLATION.name())) {
            selectedMessages.add(message);
            continue;
        }
        String fromState = message.getFromState();
        String toState = message.getToState();
        String transition = fromState + "-" + toState;
        int priority = Integer.MAX_VALUE;
        if (stateTransitionPriorities.containsKey(transition)) {
            priority = stateTransitionPriorities.get(transition);
        }
        if (!messagesGroupByStateTransitPriority.containsKey(priority)) {
            messagesGroupByStateTransitPriority.put(priority, new ArrayList<Message>());
        }
        messagesGroupByStateTransitPriority.get(priority).add(message);
        if (fromState.equals(stateModelDef.getTopState())) {
            fromTopStateMessages.add(message);
        }
    }
    // select messages
    for (List<Message> messageList : messagesGroupByStateTransitPriority.values()) {
        for (Message message : messageList) {
            String toState = message.getToState();
            if (stateConstraints.containsKey(toState)) {
                int newCnt = (stateCnts.containsKey(toState) ? stateCnts.get(toState) + 1 : 1);
                if (newCnt > stateConstraints.get(toState).getUpperBound()) {
                    if (p2pMessageEnabled && toState.equals(stateModelDef.getTopState()) && stateModelDef.isSingleTopStateModel()) {
                        // attach this message as a relay message to the message to transition off current top-state replica
                        if (fromTopStateMessages.size() > 0) {
                            Message fromTopStateMsg = fromTopStateMessages.get(0);
                            fromTopStateMsg.attachRelayMessage(message.getTgtName(), message);
                            fromTopStateMessages.remove(0);
                        }
                    } else {
                        // reach upper-bound of message for the topState, will not send the message
                        LOG.info("Reach upper_bound: " + stateConstraints.get(toState).getUpperBound() + ", not send message: " + message);
                    }
                    continue;
                }
            }
            increaseStateCnt(stateConstraints, message.getToState(), stateCnts);
            selectedMessages.add(message);
        }
    }
    return selectedMessages;
}

Example 99 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class MessageThrottleStage method throttle.

private List<Message> throttle(Map<String, Integer> throttleMap, ClusterConstraints constraint, List<Message> messages, final boolean needThrottle) {
    List<Message> throttleOutputMsgs = new ArrayList<Message>();
    for (Message message : messages) {
        Map<ConstraintAttribute, String> msgAttr = ClusterConstraints.toConstraintAttributes(message);
        Set<ConstraintItem> matches = constraint.match(msgAttr);
        matches = selectConstraints(matches, msgAttr);
        boolean msgThrottled = false;
        for (ConstraintItem item : matches) {
            String key = item.filter(msgAttr).toString();
            if (!throttleMap.containsKey(key)) {
                throttleMap.put(key, valueOf(item.getConstraintValue()));
            }
            int value = throttleMap.get(key);
            throttleMap.put(key, --value);
            if (needThrottle && value < 0) {
                msgThrottled = true;
                if (LOG.isDebugEnabled()) {
                    // TODO: printout constraint item that throttles the message
                    LOG.debug("message: " + message + " is throttled by constraint: " + item);
                }
            }
        }
        if (!msgThrottled) {
            throttleOutputMsgs.add(message);
        }
    }
    return throttleOutputMsgs;
}

Example 100 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class TaskAssignmentStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    MessageThrottleStageOutput messageOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, LiveInstance> liveInstanceMap = cache.getLiveInstances();
    if (manager == null || resourceMap == null || messageOutput == null || cache == null || liveInstanceMap == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|RESOURCES|MESSAGES_THROTTLE|DataCache|liveInstanceMap");
    }
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    List<Message> messagesToSend = new ArrayList<Message>();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        for (Partition partition : resource.getPartitions()) {
            List<Message> messages = messageOutput.getMessages(resourceName, partition);
            messagesToSend.addAll(messages);
        }
    }
    List<Message> outputMessages = batchMessage(dataAccessor.keyBuilder(), messagesToSend, resourceMap, liveInstanceMap, manager.getProperties());
    sendMessages(dataAccessor, outputMessages);
    // TODO: Need also count messages from task rebalancer
    if (!cache.isTaskCache()) {
        ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        if (clusterStatusMonitor != null) {
            clusterStatusMonitor.increaseMessageReceived(outputMessages);
        }
    }
    long cacheStart = System.currentTimeMillis();
    cache.cacheMessages(outputMessages);
    long cacheEnd = System.currentTimeMillis();
    logger.debug("Caching messages took " + (cacheEnd - cacheStart) + " ms");
}