Examples with SubscriptionData org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData used on opensource projects

Example 51 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPullConsumerImpl method pullAsyncImpl.

private // 
void pullAsyncImpl(// 
final MessageQueue mq, // 
final String subExpression, // 
final long offset, // 
final int maxNums, // 
final PullCallback pullCallback, // 
final boolean block, final long timeout) throws MQClientException, RemotingException, InterruptedException {
    this.makeSureStateOK();
    if (null == mq) {
        throw new MQClientException("mq is null", null);
    }
    if (offset < 0) {
        throw new MQClientException("offset < 0", null);
    }
    if (maxNums <= 0) {
        throw new MQClientException("maxNums <= 0", null);
    }
    if (null == pullCallback) {
        throw new MQClientException("pullCallback is null", null);
    }
    this.subscriptionAutomatically(mq.getTopic());
    try {
        int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
        final SubscriptionData subscriptionData;
        try {
            subscriptionData = // 
            FilterAPI.buildSubscriptionData(// 
            this.defaultMQPullConsumer.getConsumerGroup(), mq.getTopic(), subExpression);
        } catch (Exception e) {
            throw new MQClientException("parse subscription error", e);
        }
        long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
        // 
        this.pullAPIWrapper.pullKernelImpl(// 1
        mq, // 2
        subscriptionData.getSubString(), // 3
        0L, // 4
        offset, // 5
        maxNums, // 6
        sysFlag, // 7
        0, // 8
        this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 9
        timeoutMillis, // 10
        CommunicationMode.ASYNC, new PullCallback() {

            @Override
            public void onSuccess(PullResult pullResult) {
                pullCallback.onSuccess(DefaultMQPullConsumerImpl.this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData));
            }

            @Override
            public void onException(Throwable e) {
                pullCallback.onException(e);
            }
        });
    } catch (MQBrokerException e) {
        throw new MQClientException("pullAsync unknow exception", e);
    }
}

Example 52 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPullConsumerImpl method subscriptions.

@Override
public Set<SubscriptionData> subscriptions() {
    Set<SubscriptionData> result = new HashSet<SubscriptionData>();
    Set<String> topics = this.defaultMQPullConsumer.getRegisterTopics();
    if (topics != null) {
        synchronized (topics) {
            for (String t : topics) {
                SubscriptionData ms = null;
                try {
                    ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL);
                } catch (Exception e) {
                    log.error("parse subscription error", e);
                }
                ms.setSubVersion(0L);
                result.add(ms);
            }
        }
    }
    return result;
}

Example 53 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPullConsumerImpl method pullSyncImpl.

private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
    this.makeSureStateOK();
    if (null == mq) {
        throw new MQClientException("mq is null", null);
    }
    if (offset < 0) {
        throw new MQClientException("offset < 0", null);
    }
    if (maxNums <= 0) {
        throw new MQClientException("maxNums <= 0", null);
    }
    this.subscriptionAutomatically(mq.getTopic());
    int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
    SubscriptionData subscriptionData;
    try {
        subscriptionData = // 
        FilterAPI.buildSubscriptionData(// 
        this.defaultMQPullConsumer.getConsumerGroup(), mq.getTopic(), subExpression);
    } catch (Exception e) {
        throw new MQClientException("parse subscription error", e);
    }
    long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
    PullResult pullResult = // 
    this.pullAPIWrapper.pullKernelImpl(// 1
    mq, // 2
    subscriptionData.getSubString(), // 3
    0L, // 4
    offset, // 5
    maxNums, // 6
    sysFlag, // 7
    0, // 8
    this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 9
    timeoutMillis, // 10
    CommunicationMode.SYNC, // 11
    null);
    this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);
    if (!this.consumeMessageHookList.isEmpty()) {
        ConsumeMessageContext consumeMessageContext = null;
        consumeMessageContext = new ConsumeMessageContext();
        consumeMessageContext.setConsumerGroup(this.groupName());
        consumeMessageContext.setMq(mq);
        consumeMessageContext.setMsgList(pullResult.getMsgFoundList());
        consumeMessageContext.setSuccess(false);
        this.executeHookBefore(consumeMessageContext);
        consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString());
        consumeMessageContext.setSuccess(true);
        this.executeHookAfter(consumeMessageContext);
    }
    return pullResult;
}

Example 54 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPushConsumerImpl method copySubscription.

private void copySubscription() throws MQClientException {
    try {
        Map<String, String> sub = this.defaultMQPushConsumer.getSubscription();
        if (sub != null) {
            for (final Map.Entry<String, String> entry : sub.entrySet()) {
                final String topic = entry.getKey();
                final String subString = entry.getValue();
                SubscriptionData subscriptionData = // 
                FilterAPI.buildSubscriptionData(// 
                this.defaultMQPushConsumer.getConsumerGroup(), topic, subString);
                this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData);
            }
        }
        if (null == this.messageListenerInner) {
            this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener();
        }
        switch(this.defaultMQPushConsumer.getMessageModel()) {
            case BROADCASTING:
                break;
            case CLUSTERING:
                final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup());
                SubscriptionData subscriptionData = // 
                FilterAPI.buildSubscriptionData(// 
                this.defaultMQPushConsumer.getConsumerGroup(), retryTopic, SubscriptionData.SUB_ALL);
                this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData);
                break;
            default:
                break;
        }
    } catch (Exception e) {
        throw new MQClientException("subscription exception", e);
    }
}

Example 55 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPushConsumerImpl method pullMessage.

public void pullMessage(final PullRequest pullRequest) {
    final ProcessQueue processQueue = pullRequest.getProcessQueue();
    if (processQueue.isDropped()) {
        log.info("the pull request[{}] is dropped.", pullRequest.toString());
        return;
    }
    pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());
    try {
        this.makeSureStateOK();
    } catch (MQClientException e) {
        log.warn("pullMessage exception, consumer state not ok", e);
        // 延迟拉取请求
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        return;
    }
    if (this.isPause()) {
        log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());
        // 延迟拉取请求
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);
        return;
    }
    long size = processQueue.getMsgCount().get();
    if (size > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
        // 延迟拉取请求
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
        if ((flowControlTimes1++ % 1000) == 0) {
            log.warn("the consumer message buffer is full, so do flow control, minOffset={}, maxOffset={}, size={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), size, pullRequest, flowControlTimes1);
        }
        return;
    }
    if (!this.consumeOrderly) {
        if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {
            // 延迟拉取请求
            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
            if ((flowControlTimes2++ % 1000) == 0) {
                log.warn("the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), pullRequest, flowControlTimes2);
            }
            return;
        }
    } else {
        if (processQueue.isLocked()) {
            if (!pullRequest.isLockedFirst()) {
                final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue());
                boolean brokerBusy = offset < pullRequest.getNextOffset();
                log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy);
                if (brokerBusy) {
                    log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", pullRequest, offset);
                }
                pullRequest.setLockedFirst(true);
                pullRequest.setNextOffset(offset);
            }
        } else {
            // 延迟拉取请求
            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
            log.info("pull message later because not locked in broker, {}", pullRequest);
            return;
        }
    }
    final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
    if (null == subscriptionData) {
        // 延迟拉取请求
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        log.warn("find the consumer's subscription failed, {}", pullRequest);
        return;
    }
    final long beginTimestamp = System.currentTimeMillis();
    PullCallback pullCallback = new PullCallback() {

        @Override
        public void onSuccess(PullResult pullResult) {
            if (pullResult != null) {
                pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData);
                switch(pullResult.getPullStatus()) {
                    case FOUND:
                        long prevRequestOffset = pullRequest.getNextOffset();
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        long pullRT = System.currentTimeMillis() - beginTimestamp;
                        DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullRT);
                        long firstMsgOffset = Long.MAX_VALUE;
                        if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
                            // 马上执行拉请求
                            DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        } else {
                            firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
                            DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
                            boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
                            // 
                            DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(// 
                            pullResult.getMsgFoundList(), // 
                            processQueue, // 
                            pullRequest.getMessageQueue(), dispathToConsume);
                            if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
                                // 延迟拉取请求
                                DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
                            } else {
                                // 马上执行拉请求
                                DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                            }
                        }
                        if (// 
                        pullResult.getNextBeginOffset() < prevRequestOffset || firstMsgOffset < prevRequestOffset) {
                            log.warn(// 
                            "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", // 
                            pullResult.getNextBeginOffset(), // 
                            firstMsgOffset, prevRequestOffset);
                        }
                        break;
                    case NO_NEW_MSG:
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
                        // 马上执行拉请求
                        DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        break;
                    case NO_MATCHED_MSG:
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
                        // 马上执行拉请求
                        DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        break;
                    case OFFSET_ILLEGAL:
                        // 
                        log.warn(// 
                        "the pull request offset illegal, {} {}", pullRequest.toString(), pullResult.toString());
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        pullRequest.getProcessQueue().setDropped(true);
                        DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() {

                            @Override
                            public void run() {
                                try {
                                    DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), false);
                                    DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
                                    DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
                                    log.warn("fix the pull request offset, {}", pullRequest);
                                } catch (Throwable e) {
                                    log.error("executeTaskLater Exception", e);
                                }
                            }
                        }, 10000);
                        break;
                    default:
                        break;
                }
            }
        }

        @Override
        public void onException(Throwable e) {
            if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                log.warn("execute the pull request exception", e);
            }
            // 延迟拉取请求
            DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        }
    };
    boolean commitOffsetEnable = false;
    long commitOffsetValue = 0L;
    if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {
        commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);
        if (commitOffsetValue > 0) {
            commitOffsetEnable = true;
        }
    }
    String subExpression = null;
    boolean classFilter = false;
    SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
    if (sd != null) {
        if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) {
            subExpression = sd.getSubString();
        }
        classFilter = sd.isClassFilterMode();
    }
    int sysFlag = // 
    PullSysFlag.buildSysFlag(// commitOffset
    commitOffsetEnable, // suspend
    true, // subscription
    subExpression != null, // class filter
    classFilter);
    try {
        // 
        this.pullAPIWrapper.pullKernelImpl(// 1
        pullRequest.getMessageQueue(), // 2
        subExpression, // 3
        subscriptionData.getExpressionType(), // 4
        subscriptionData.getSubVersion(), // 5
        pullRequest.getNextOffset(), // 6  一次拉多少
        this.defaultMQPushConsumer.getPullBatchSize(), // 7
        sysFlag, // 8
        commitOffsetValue, // 9
        BROKER_SUSPEND_MAX_TIME_MILLIS, // 10
        CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 11
        CommunicationMode.ASYNC, // 12
        pullCallback);
    } catch (Exception e) {
        log.error("pullKernelImpl exception", e);
        // 延迟拉取请求
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
    }
}