Examples with PreFetch org.apache.helix.api.listeners.PreFetch used on opensource projects

Example 1 with PreFetch

use of org.apache.helix.api.listeners.PreFetch in project helix by apache.

the class HelixTaskExecutor method onMessage.

@Override
@PreFetch(enabled = false)
public void onMessage(String instanceName, List<Message> messages, NotificationContext changeContext) {
    HelixManager manager = changeContext.getManager();
    // TODO: see if we should have a separate notification call for resetting
    if (changeContext.getType() == Type.FINALIZE) {
        reset();
        return;
    }
    if (changeContext.getType() == Type.INIT) {
        init();
    // continue to process messages
    }
    // if prefetch is disabled in MessageListenerCallback, we need to read all new messages from zk.
    if (messages == null || messages.isEmpty()) {
        // If no messages are given, check and read all new messages.
        messages = readNewMessagesFromZK(manager, instanceName, changeContext.getChangeType());
    }
    if (_isShuttingDown) {
        StringBuilder sb = new StringBuilder();
        for (Message message : messages) {
            sb.append(message.getMsgId() + ",");
        }
        LOG.info("Helix task executor is shutting down, discard unprocessed messages : " + sb.toString());
        return;
    }
    // Update message count
    if (_messageQueueMonitor != null) {
        _messageQueueMonitor.setMessageQueueBacklog(messages.size());
    }
    if (messages.isEmpty()) {
        LOG.info("No Messages to process");
        return;
    }
    // sort message by creation timestamp, so message created earlier is processed first
    Collections.sort(messages, Message.CREATE_TIME_COMPARATOR);
    HelixDataAccessor accessor = manager.getHelixDataAccessor();
    Builder keyBuilder = accessor.keyBuilder();
    // message handlers created
    Map<String, MessageHandler> stateTransitionHandlers = new HashMap<>();
    List<MessageHandler> nonStateTransitionHandlers = new ArrayList<>();
    // message read
    List<Message> readMsgs = new ArrayList<>();
    String sessionId = manager.getSessionId();
    List<String> curResourceNames = accessor.getChildNames(keyBuilder.currentStates(instanceName, sessionId));
    List<PropertyKey> createCurStateKeys = new ArrayList<>();
    List<CurrentState> metaCurStates = new ArrayList<>();
    Set<String> createCurStateNames = new HashSet<>();
    for (Message message : messages) {
        // situations such as register a new message handler factory
        if (message.getMsgType().equalsIgnoreCase(MessageType.NO_OP.toString())) {
            LOG.info("Dropping NO-OP message. mid: " + message.getId() + ", from: " + message.getMsgSrc());
            reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
            continue;
        }
        if (message.isExpired()) {
            LOG.info("Dropping expired message. mid: " + message.getId() + ", from: " + message.getMsgSrc() + " relayed from: " + message.getRelaySrcHost());
            reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
            continue;
        }
        String tgtSessionId = message.getTgtSessionId();
        // sessionId mismatch normally means message comes from expired session, just remove it
        if (!sessionId.equals(tgtSessionId) && !tgtSessionId.equals("*")) {
            String warningMessage = "SessionId does NOT match. expected sessionId: " + sessionId + ", tgtSessionId in message: " + tgtSessionId + ", messageId: " + message.getMsgId();
            LOG.warn(warningMessage);
            reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.DISCARDED);
            _statusUpdateUtil.logWarning(message, HelixStateMachineEngine.class, warningMessage, manager);
            // upon session mismatch after a new session is established
            if (manager.getInstanceType() == InstanceType.PARTICIPANT || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) {
                if (message.getCreateTimeStamp() > manager.getSessionStartTime()) {
                    syncSessionToController(manager);
                }
            }
            continue;
        }
        if ((manager.getInstanceType() == InstanceType.CONTROLLER || manager.getInstanceType() == InstanceType.CONTROLLER_PARTICIPANT) && MessageType.PARTICIPANT_SESSION_CHANGE.name().equals(message.getMsgType())) {
            LOG.info(String.format("Controller received PARTICIPANT_SESSION_CHANGE msg from src: %s", message.getMsgSrc()));
            PropertyKey key = new Builder(manager.getClusterName()).liveInstances();
            List<LiveInstance> liveInstances = manager.getHelixDataAccessor().getChildValues(key);
            _controller.onLiveInstanceChange(liveInstances, changeContext);
            reportAndRemoveMessage(message, accessor, instanceName, ProcessedMessageState.COMPLETED);
            continue;
        }
        // don't process message that is of READ or UNPROCESSABLE state
        if (MessageState.NEW != message.getMsgState()) {
            // check for the status and ignore if its already read
            if (LOG.isTraceEnabled()) {
                LOG.trace("Message already read. msgId: " + message.getMsgId());
            }
            continue;
        }
        // State Transition Cancellation
        if (message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
            boolean success = cancelNotStartedStateTransition(message, stateTransitionHandlers, accessor, instanceName);
            if (success) {
                continue;
            }
        }
        _monitor.reportReceivedMessage(message);
        // create message handlers, if handlers not found, leave its state as NEW
        try {
            MessageHandler createHandler = createMessageHandler(message, changeContext);
            if (createHandler == null) {
                continue;
            }
            if (message.getMsgType().equals(MessageType.STATE_TRANSITION.name()) || message.getMsgType().equals(MessageType.STATE_TRANSITION_CANCELLATION.name())) {
                stateTransitionHandlers.put(getMessageTarget(message.getResourceName(), message.getPartitionName()), createHandler);
            } else {
                nonStateTransitionHandlers.add(createHandler);
            }
        } catch (Exception e) {
            LOG.error("Failed to create message handler for " + message.getMsgId(), e);
            String error = "Failed to create message handler for " + message.getMsgId() + ", exception: " + e;
            _statusUpdateUtil.logError(message, HelixStateMachineEngine.class, e, error, manager);
            message.setMsgState(MessageState.UNPROCESSABLE);
            removeMessageFromZK(accessor, message, instanceName);
            LOG.error("Message cannot be processed: " + message.getRecord(), e);
            _monitor.reportProcessedMessage(message, ParticipantMessageMonitor.ProcessedMessageState.DISCARDED);
            continue;
        }
        markReadMessage(message, changeContext, manager);
        readMsgs.add(message);
        // do it for non-controller and state transition messages only
        if (!message.isControlerMsg() && message.getMsgType().equals(Message.MessageType.STATE_TRANSITION.name())) {
            String resourceName = message.getResourceName();
            if (!curResourceNames.contains(resourceName) && !createCurStateNames.contains(resourceName)) {
                createCurStateNames.add(resourceName);
                createCurStateKeys.add(keyBuilder.currentState(instanceName, sessionId, resourceName));
                CurrentState metaCurState = new CurrentState(resourceName);
                metaCurState.setBucketSize(message.getBucketSize());
                metaCurState.setStateModelDefRef(message.getStateModelDef());
                metaCurState.setSessionId(sessionId);
                metaCurState.setBatchMessageMode(message.getBatchMessageMode());
                String ftyName = message.getStateModelFactoryName();
                if (ftyName != null) {
                    metaCurState.setStateModelFactoryName(ftyName);
                } else {
                    metaCurState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
                }
                metaCurStates.add(metaCurState);
            }
        }
    }
    // batch create curState meta
    if (createCurStateKeys.size() > 0) {
        try {
            accessor.createChildren(createCurStateKeys, metaCurStates);
        } catch (Exception e) {
            LOG.error("fail to create cur-state znodes for messages: " + readMsgs, e);
        }
    }
    // update message state to READ in batch and schedule all read messages
    if (readMsgs.size() > 0) {
        updateMessageState(readMsgs, accessor, instanceName);
        for (MessageHandler handler : stateTransitionHandlers.values()) {
            HelixTask task = new HelixTask(handler._message, changeContext, handler, this);
            scheduleTask(task);
        }
        for (MessageHandler handler : nonStateTransitionHandlers) {
            HelixTask task = new HelixTask(handler._message, changeContext, handler, this);
            scheduleTask(task);
        }
    }
}

Example 2 with PreFetch

use of org.apache.helix.api.listeners.PreFetch in project helix by apache.

the class RoutingTableProvider method onExternalViewChange.

@Override
@PreFetch(enabled = false)
public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) {
    HelixConstants.ChangeType changeType = changeContext.getChangeType();
    if (changeType != null && !changeType.getPropertyType().equals(_sourceDataType)) {
        logger.warn("onExternalViewChange called with dis-matched change types. Source data type " + _sourceDataType + ", changed data type: " + changeType);
        return;
    }
    // Refresh with full list of external view.
    if (externalViewList != null && externalViewList.size() > 0) {
        // keep this here for back-compatibility, application can call onExternalViewChange directly with externalview list supplied.
        refresh(externalViewList, changeContext);
    } else {
        ClusterEventType eventType;
        if (_sourceDataType.equals(PropertyType.EXTERNALVIEW)) {
            eventType = ClusterEventType.ExternalViewChange;
        } else if (_sourceDataType.equals(PropertyType.TARGETEXTERNALVIEW)) {
            eventType = ClusterEventType.TargetExternalViewChange;
        } else {
            logger.warn("onExternalViewChange called with dis-matched change types. Source data type " + _sourceDataType + ", change type: " + changeType);
            return;
        }
        _routerUpdater.queueEvent(changeContext, eventType, changeType);
    }
}

Example 3 with PreFetch

use of org.apache.helix.api.listeners.PreFetch in project helix by apache.

the class ZkClient method isPrefetchEnabled.

private boolean isPrefetchEnabled(IZkDataListener dataListener) {
    PreFetch preFetch = dataListener.getClass().getAnnotation(PreFetch.class);
    if (preFetch != null) {
        return preFetch.enabled();
    }
    Method callbackMethod = IZkDataListener.class.getMethods()[0];
    try {
        Method method = dataListener.getClass().getMethod(callbackMethod.getName(), callbackMethod.getParameterTypes());
        PreFetch preFetchInMethod = method.getAnnotation(PreFetch.class);
        if (preFetchInMethod != null) {
            return preFetchInMethod.enabled();
        }
    } catch (NoSuchMethodException e) {
        LOG.warn("No method " + callbackMethod.getName() + " defined in listener " + dataListener.getClass().getCanonicalName());
    }
    return true;
}

Example 4 with PreFetch

use of org.apache.helix.api.listeners.PreFetch in project helix by apache.

the class CallbackHandler method parseListenerProperties.

private void parseListenerProperties() {
    BatchMode batchMode = _listener.getClass().getAnnotation(BatchMode.class);
    PreFetch preFetch = _listener.getClass().getAnnotation(PreFetch.class);
    String asyncBatchModeEnabled = System.getProperty("helix.callbackhandler.isAsyncBatchModeEnabled");
    if (asyncBatchModeEnabled == null) {
        // for backcompatible, the old property name is deprecated.
        asyncBatchModeEnabled = System.getProperty("isAsyncBatchModeEnabled");
    }
    if (asyncBatchModeEnabled != null) {
        _batchModeEnabled = Boolean.parseBoolean(asyncBatchModeEnabled);
        logger.info("isAsyncBatchModeEnabled by default: " + _batchModeEnabled);
    }
    if (batchMode != null) {
        _batchModeEnabled = batchMode.enabled();
    }
    if (preFetch != null) {
        _preFetchEnabled = preFetch.enabled();
    }
    Class listenerClass = null;
    switch(_changeType) {
        case IDEAL_STATE:
            listenerClass = IdealStateChangeListener.class;
            break;
        case INSTANCE_CONFIG:
            if (_listener instanceof ConfigChangeListener) {
                listenerClass = ConfigChangeListener.class;
            } else if (_listener instanceof InstanceConfigChangeListener) {
                listenerClass = InstanceConfigChangeListener.class;
            }
            break;
        case CLUSTER_CONFIG:
            listenerClass = ClusterConfigChangeListener.class;
            break;
        case RESOURCE_CONFIG:
            listenerClass = ResourceConfigChangeListener.class;
            break;
        case CONFIG:
            listenerClass = ConfigChangeListener.class;
            break;
        case LIVE_INSTANCE:
            listenerClass = LiveInstanceChangeListener.class;
            break;
        case CURRENT_STATE:
            listenerClass = CurrentStateChangeListener.class;
            ;
            break;
        case MESSAGE:
        case MESSAGES_CONTROLLER:
            listenerClass = MessageListener.class;
            break;
        case EXTERNAL_VIEW:
        case TARGET_EXTERNAL_VIEW:
            listenerClass = ExternalViewChangeListener.class;
            break;
        case CONTROLLER:
            listenerClass = ControllerChangeListener.class;
    }
    Method callbackMethod = listenerClass.getMethods()[0];
    try {
        Method method = _listener.getClass().getMethod(callbackMethod.getName(), callbackMethod.getParameterTypes());
        BatchMode batchModeInMethod = method.getAnnotation(BatchMode.class);
        PreFetch preFetchInMethod = method.getAnnotation(PreFetch.class);
        if (batchModeInMethod != null) {
            _batchModeEnabled = batchModeInMethod.enabled();
        }
        if (preFetchInMethod != null) {
            _preFetchEnabled = preFetchInMethod.enabled();
        }
    } catch (NoSuchMethodException e) {
        logger.warn("No method " + callbackMethod.getName() + " defined in listener " + _listener.getClass().getCanonicalName());
    }
}