Examples with Message org.apache.helix.model.Message used on opensource projects

Example 1 with Message

use of org.apache.helix.model.Message in project incubator-gobblin by apache.

the class GobblinHelixMessagingService method generateMessage.

@Override
public Map<InstanceType, List<Message>> generateMessage(final Criteria recipientCriteria, final Message message) {
    Map<InstanceType, List<Message>> messagesToSendMap = new HashMap<InstanceType, List<Message>>();
    InstanceType instanceType = recipientCriteria.getRecipientInstanceType();
    if (instanceType == InstanceType.CONTROLLER) {
        List<Message> messages = generateMessagesForController(message);
        messagesToSendMap.put(InstanceType.CONTROLLER, messages);
    // _dataAccessor.setControllerProperty(PropertyType.MESSAGES,
    // newMessage.getRecord(), CreateMode.PERSISTENT);
    } else if (instanceType == InstanceType.PARTICIPANT) {
        List<Message> messages = new ArrayList<Message>();
        List<Map<String, String>> matchedList = _gobblinHelixCriteriaEvaluator.evaluateCriteria(recipientCriteria, _manager);
        if (!matchedList.isEmpty()) {
            Map<String, String> sessionIdMap = new HashMap<String, String>();
            if (recipientCriteria.isSessionSpecific()) {
                HelixDataAccessor accessor = _manager.getHelixDataAccessor();
                PropertyKey.Builder keyBuilder = accessor.keyBuilder();
                List<LiveInstance> liveInstances = accessor.getChildValues(keyBuilder.liveInstances());
                for (LiveInstance liveInstance : liveInstances) {
                    sessionIdMap.put(liveInstance.getInstanceName(), liveInstance.getSessionId());
                }
            }
            for (Map<String, String> map : matchedList) {
                String id = UUID.randomUUID().toString();
                Message newMessage = new Message(message.getRecord(), id);
                String srcInstanceName = _manager.getInstanceName();
                String tgtInstanceName = map.get("instanceName");
                // Don't send message to self
                if (recipientCriteria.isSelfExcluded() && srcInstanceName.equalsIgnoreCase(tgtInstanceName)) {
                    continue;
                }
                newMessage.setSrcName(srcInstanceName);
                newMessage.setTgtName(tgtInstanceName);
                newMessage.setResourceName(map.get("resourceName"));
                newMessage.setPartitionName(map.get("partitionName"));
                if (recipientCriteria.isSessionSpecific()) {
                    newMessage.setTgtSessionId(sessionIdMap.get(tgtInstanceName));
                }
                messages.add(newMessage);
            }
            messagesToSendMap.put(InstanceType.PARTICIPANT, messages);
        }
    }
    return messagesToSendMap;
}

Example 2 with Message

use of org.apache.helix.model.Message in project incubator-gobblin by apache.

the class GobblinHelixMessagingService method generateMessagesForController.

private List<Message> generateMessagesForController(Message message) {
    List<Message> messages = new ArrayList<Message>();
    String id = UUID.randomUUID().toString();
    Message newMessage = new Message(message.getRecord(), id);
    newMessage.setMsgId(id);
    newMessage.setSrcName(_manager.getInstanceName());
    newMessage.setTgtName("Controller");
    messages.add(newMessage);
    return messages;
}

Example 3 with Message

use of org.apache.helix.model.Message in project incubator-gobblin by apache.

the class GobblinYarnAppLauncher method sendShutdownRequest.

@VisibleForTesting
void sendShutdownRequest() {
    Criteria criteria = new Criteria();
    criteria.setInstanceName("%");
    criteria.setResource("%");
    criteria.setPartition("%");
    criteria.setPartitionState("%");
    criteria.setRecipientInstanceType(InstanceType.CONTROLLER);
    criteria.setSessionSpecific(true);
    Message shutdownRequest = new Message(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString().toLowerCase() + UUID.randomUUID().toString());
    shutdownRequest.setMsgSubType(HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString());
    shutdownRequest.setMsgState(Message.MessageState.NEW);
    shutdownRequest.setTgtSessionId("*");
    int messagesSent = this.helixManager.getMessagingService().send(criteria, shutdownRequest);
    if (messagesSent == 0) {
        LOGGER.error(String.format("Failed to send the %s message to the controller", shutdownRequest.getMsgSubType()));
    }
}

Example 4 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class MessageGenerationPhase method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES_TO_REBALANCE.name());
    Map<String, List<Message>> pendingMessagesToCleanUp = new HashMap<>();
    CurrentStateOutput currentStateOutput = event.getAttribute(AttributeName.CURRENT_STATE.name());
    IntermediateStateOutput intermediateStateOutput = event.getAttribute(AttributeName.INTERMEDIATE_STATE.name());
    if (manager == null || cache == null || resourceMap == null || currentStateOutput == null || intermediateStateOutput == null) {
        throw new StageException("Missing attributes in event:" + event + ". Requires HelixManager|DataCache|RESOURCES|CURRENT_STATE|INTERMEDIATE_STATE");
    }
    Map<String, LiveInstance> liveInstances = cache.getLiveInstances();
    Map<String, String> sessionIdMap = new HashMap<String, String>();
    for (LiveInstance liveInstance : liveInstances.values()) {
        sessionIdMap.put(liveInstance.getInstanceName(), liveInstance.getSessionId());
    }
    MessageGenerationOutput output = new MessageGenerationOutput();
    for (String resourceName : resourceMap.keySet()) {
        Resource resource = resourceMap.get(resourceName);
        StateModelDefinition stateModelDef = cache.getStateModelDef(resource.getStateModelDefRef());
        if (stateModelDef == null) {
            logger.error("State Model Definition null, skip generating messages for resource: " + resourceName);
            continue;
        }
        for (Partition partition : resource.getPartitions()) {
            Map<String, String> instanceStateMap = new HashMap<String, String>(intermediateStateOutput.getInstanceStateMap(resourceName, partition));
            Map<String, String> pendingStateMap = currentStateOutput.getPendingStateMap(resourceName, partition);
            for (String instance : pendingStateMap.keySet()) {
                if (!instanceStateMap.containsKey(instance)) {
                    instanceStateMap.put(instance, NO_DESIRED_STATE);
                }
            }
            // we should generate message based on the desired-state priority
            // so keep generated messages in a temp map keyed by state
            // desired-state->list of generated-messages
            Map<String, List<Message>> messageMap = new HashMap<String, List<Message>>();
            for (String instanceName : instanceStateMap.keySet()) {
                String desiredState = instanceStateMap.get(instanceName);
                String currentState = currentStateOutput.getCurrentState(resourceName, partition, instanceName);
                if (currentState == null) {
                    currentState = stateModelDef.getInitialState();
                }
                Message pendingMessage = currentStateOutput.getPendingState(resourceName, partition, instanceName);
                boolean isCancellationEnabled = cache.getClusterConfig().isStateTransitionCancelEnabled();
                Message cancellationMessage = currentStateOutput.getCancellationState(resourceName, partition, instanceName);
                String nextState = stateModelDef.getNextStateForTransition(currentState, desiredState);
                Message message = null;
                if (shouldCleanUpPendingMessage(pendingMessage, currentState, currentStateOutput.getEndTime(resourceName, partition, instanceName))) {
                    logger.info("Adding pending message {} on instance {} to clean up. Msg: {}->{}, current state of resource {}:{} is {}", pendingMessage.getMsgId(), instanceName, pendingMessage.getFromState(), pendingMessage.getToState(), resourceName, partition, currentState);
                    if (!pendingMessagesToCleanUp.containsKey(instanceName)) {
                        pendingMessagesToCleanUp.put(instanceName, new ArrayList<Message>());
                    }
                    pendingMessagesToCleanUp.get(instanceName).add(pendingMessage);
                }
                if (desiredState.equals(NO_DESIRED_STATE) || desiredState.equalsIgnoreCase(currentState)) {
                    if (desiredState.equals(NO_DESIRED_STATE) || pendingMessage != null && !currentState.equalsIgnoreCase(pendingMessage.getToState())) {
                        message = createStateTransitionCancellationMessage(manager, resource, partition.getPartitionName(), instanceName, sessionIdMap.get(instanceName), stateModelDef.getId(), pendingMessage.getFromState(), pendingMessage.getToState(), null, cancellationMessage, isCancellationEnabled, currentState);
                    }
                } else {
                    if (nextState == null) {
                        logger.error("Unable to find a next state for resource: " + resource.getResourceName() + " partition: " + partition.getPartitionName() + " from stateModelDefinition" + stateModelDef.getClass() + " from:" + currentState + " to:" + desiredState);
                        continue;
                    }
                    if (pendingMessage != null) {
                        String pendingState = pendingMessage.getToState();
                        if (nextState.equalsIgnoreCase(pendingState)) {
                            logger.debug("Message already exists for " + instanceName + " to transit " + resource.getResourceName() + "." + partition.getPartitionName() + " from " + currentState + " to " + nextState);
                        } else if (currentState.equalsIgnoreCase(pendingState)) {
                            logger.info("Message hasn't been removed for " + instanceName + " to transit " + resource.getResourceName() + "." + partition.getPartitionName() + " to " + pendingState + ", desiredState: " + desiredState);
                        } else {
                            logger.info("IdealState changed before state transition completes for " + resource.getResourceName() + "." + partition.getPartitionName() + " on " + instanceName + ", pendingState: " + pendingState + ", currentState: " + currentState + ", nextState: " + nextState);
                            message = createStateTransitionCancellationMessage(manager, resource, partition.getPartitionName(), instanceName, sessionIdMap.get(instanceName), stateModelDef.getId(), pendingMessage.getFromState(), pendingState, nextState, cancellationMessage, isCancellationEnabled, currentState);
                        }
                    } else {
                        // Create new state transition message
                        message = createStateTransitionMessage(manager, resource, partition.getPartitionName(), instanceName, currentState, nextState, sessionIdMap.get(instanceName), stateModelDef.getId());
                    }
                }
                if (message != null) {
                    IdealState idealState = cache.getIdealState(resourceName);
                    if (idealState != null && idealState.getStateModelDefRef().equalsIgnoreCase(DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE)) {
                        if (idealState.getRecord().getMapField(partition.getPartitionName()) != null) {
                            message.getRecord().setMapField(Message.Attributes.INNER_MESSAGE.toString(), idealState.getRecord().getMapField(partition.getPartitionName()));
                        }
                    }
                    int timeout = getTimeOut(cache.getClusterConfig(), cache.getResourceConfig(resourceName), currentState, nextState, idealState, partition);
                    if (timeout > 0) {
                        message.setExecutionTimeout(timeout);
                    }
                    message.setAttribute(Message.Attributes.ClusterEventName, event.getEventType().name());
                    // output.addMessage(resourceName, partition, message);
                    if (!messageMap.containsKey(desiredState)) {
                        messageMap.put(desiredState, new ArrayList<Message>());
                    }
                    messageMap.get(desiredState).add(message);
                }
            }
            // add generated messages to output according to state priority
            List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
            for (String state : statesPriorityList) {
                if (messageMap.containsKey(state)) {
                    for (Message message : messageMap.get(state)) {
                        output.addMessage(resourceName, partition, message);
                    }
                }
            }
        }
    // end of for-each-partition
    }
    // Asynchronously clean up pending messages if necessary
    if (!pendingMessagesToCleanUp.isEmpty()) {
        schedulePendingMessageCleanUp(pendingMessagesToCleanUp, cache.getAsyncTasksThreadPool(), manager.getHelixDataAccessor());
    }
    event.addAttribute(AttributeName.MESSAGES_ALL.name(), output);
}

Example 5 with Message

use of org.apache.helix.model.Message in project helix by apache.

the class MessageGenerationPhase method createStateTransitionMessage.

private Message createStateTransitionMessage(HelixManager manager, Resource resource, String partitionName, String instanceName, String currentState, String nextState, String sessionId, String stateModelDefName) {
    String uuid = UUID.randomUUID().toString();
    Message message = new Message(MessageType.STATE_TRANSITION, uuid);
    message.setSrcName(manager.getInstanceName());
    message.setTgtName(instanceName);
    message.setMsgState(MessageState.NEW);
    message.setPartitionName(partitionName);
    message.setResourceName(resource.getResourceName());
    message.setFromState(currentState);
    message.setToState(nextState);
    message.setTgtSessionId(sessionId);
    message.setSrcSessionId(manager.getSessionId());
    message.setStateModelDef(stateModelDefName);
    message.setStateModelFactoryName(resource.getStateModelFactoryname());
    message.setBucketSize(resource.getBucketSize());
    if (resource.getResourceGroupName() != null) {
        message.setResourceGroupName(resource.getResourceGroupName());
    }
    if (resource.getResourceTag() != null) {
        message.setResourceTag(resource.getResourceTag());
    }
    return message;
}