Examples with PullResult org.apache.rocketmq.client.consumer.PullResult used on opensource projects

Example 26 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class PullConsumerDemo method main.

public static void main(String[] args) throws MQClientException {
    // First we init the flow control rule for Sentinel.
    initFlowControlRule();
    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(Constants.TEST_GROUP_NAME);
    consumer.start();
    Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(Constants.TEST_TOPIC_NAME);
    for (MessageQueue mq : mqs) {
        System.out.printf("Consuming messages from the queue: %s%n", mq);
        SINGLE_MQ: while (true) {
            try {
                PullResult pullResult = consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32);
                if (pullResult.getMsgFoundList() != null) {
                    for (MessageExt msg : pullResult.getMsgFoundList()) {
                        doSomething(msg);
                    }
                }
                long nextOffset = pullResult.getNextBeginOffset();
                putMessageQueueOffset(mq, nextOffset);
                consumer.updateConsumeOffset(mq, nextOffset);
                switch(pullResult.getPullStatus()) {
                    case FOUND:
                        break;
                    case NO_MATCHED_MSG:
                        break;
                    case NO_NEW_MSG:
                        break SINGLE_MQ;
                    case OFFSET_ILLEGAL:
                        break;
                    default:
                        break;
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    consumer.shutdown();
}

Example 27 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class WorkerSinkTask method pullMessageFromQueues.

private void pullMessageFromQueues() throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
    long startTimeStamp = System.currentTimeMillis();
    log.info("START pullMessageFromQueues, time started : {}", startTimeStamp);
    for (Map.Entry<MessageQueue, Long> entry : messageQueuesOffsetMap.entrySet()) {
        if (messageQueuesStateMap.containsKey(entry.getKey())) {
            continue;
        }
        log.info("START pullBlockIfNotFound, time started : {}", System.currentTimeMillis());
        if (WorkerTaskState.RUNNING != state.get()) {
            break;
        }
        final PullResult pullResult = consumer.pullBlockIfNotFound(entry.getKey(), "*", entry.getValue(), MAX_MESSAGE_NUM);
        long currentTime = System.currentTimeMillis();
        log.info("INSIDE pullMessageFromQueues, time elapsed : {}", currentTime - startTimeStamp);
        if (pullResult.getPullStatus().equals(PullStatus.FOUND)) {
            final List<MessageExt> messages = pullResult.getMsgFoundList();
            receiveMessages(messages);
            messageQueuesOffsetMap.put(entry.getKey(), pullResult.getNextBeginOffset());
            offsetManagementService.putPosition(convertToByteBufferKey(entry.getKey()), convertToByteBufferValue(pullResult.getNextBeginOffset()));
            preCommit();
        }
    }
}

Example 28 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPullConsumerImpl method pullSyncImpl.

private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
    this.makeSureStateOK();
    if (null == mq) {
        throw new MQClientException("mq is null", null);
    }
    if (offset < 0) {
        throw new MQClientException("offset < 0", null);
    }
    if (maxNums <= 0) {
        throw new MQClientException("maxNums <= 0", null);
    }
    this.subscriptionAutomatically(mq.getTopic());
    int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
    SubscriptionData subscriptionData;
    try {
        subscriptionData = // 
        FilterAPI.buildSubscriptionData(// 
        this.defaultMQPullConsumer.getConsumerGroup(), mq.getTopic(), subExpression);
    } catch (Exception e) {
        throw new MQClientException("parse subscription error", e);
    }
    long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
    PullResult pullResult = // 
    this.pullAPIWrapper.pullKernelImpl(// 1
    mq, // 2
    subscriptionData.getSubString(), // 3
    0L, // 4
    offset, // 5
    maxNums, // 6
    sysFlag, // 7
    0, // 8
    this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 9
    timeoutMillis, // 10
    CommunicationMode.SYNC, // 11
    null);
    this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);
    if (!this.consumeMessageHookList.isEmpty()) {
        ConsumeMessageContext consumeMessageContext = null;
        consumeMessageContext = new ConsumeMessageContext();
        consumeMessageContext.setConsumerGroup(this.groupName());
        consumeMessageContext.setMq(mq);
        consumeMessageContext.setMsgList(pullResult.getMsgFoundList());
        consumeMessageContext.setSuccess(false);
        this.executeHookBefore(consumeMessageContext);
        consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString());
        consumeMessageContext.setSuccess(true);
        this.executeHookAfter(consumeMessageContext);
    }
    return pullResult;
}

Example 29 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPullConsumerImpl method pullAsyncImpl.

private // 
void pullAsyncImpl(// 
final MessageQueue mq, // 
final String subExpression, // 
final long offset, // 
final int maxNums, // 
final PullCallback pullCallback, // 
final boolean block, final long timeout) throws MQClientException, RemotingException, InterruptedException {
    this.makeSureStateOK();
    if (null == mq) {
        throw new MQClientException("mq is null", null);
    }
    if (offset < 0) {
        throw new MQClientException("offset < 0", null);
    }
    if (maxNums <= 0) {
        throw new MQClientException("maxNums <= 0", null);
    }
    if (null == pullCallback) {
        throw new MQClientException("pullCallback is null", null);
    }
    this.subscriptionAutomatically(mq.getTopic());
    try {
        int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false);
        final SubscriptionData subscriptionData;
        try {
            subscriptionData = // 
            FilterAPI.buildSubscriptionData(// 
            this.defaultMQPullConsumer.getConsumerGroup(), mq.getTopic(), subExpression);
        } catch (Exception e) {
            throw new MQClientException("parse subscription error", e);
        }
        long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout;
        // 
        this.pullAPIWrapper.pullKernelImpl(// 1
        mq, // 2
        subscriptionData.getSubString(), // 3
        0L, // 4
        offset, // 5
        maxNums, // 6
        sysFlag, // 7
        0, // 8
        this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 9
        timeoutMillis, // 10
        CommunicationMode.ASYNC, new PullCallback() {

            @Override
            public void onSuccess(PullResult pullResult) {
                pullCallback.onSuccess(DefaultMQPullConsumerImpl.this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData));
            }

            @Override
            public void onException(Throwable e) {
                pullCallback.onException(e);
            }
        });
    } catch (MQBrokerException e) {
        throw new MQClientException("pullAsync unknow exception", e);
    }
}

Example 30 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPushConsumerImpl method pullMessage.

public void pullMessage(final PullRequest pullRequest) {
    final ProcessQueue processQueue = pullRequest.getProcessQueue();
    if (processQueue.isDropped()) {
        log.info("the pull request[{}] is dropped.", pullRequest.toString());
        return;
    }
    pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());
    try {
        this.makeSureStateOK();
    } catch (MQClientException e) {
        log.warn("pullMessage exception, consumer state not ok", e);
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        return;
    }
    if (this.isPause()) {
        log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);
        return;
    }
    long size = processQueue.getMsgCount().get();
    if (size > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
        if ((flowControlTimes1++ % 1000) == 0) {
            log.warn("the consumer message buffer is full, so do flow control, minOffset={}, maxOffset={}, size={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), size, pullRequest, flowControlTimes1);
        }
        return;
    }
    if (!this.consumeOrderly) {
        if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {
            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
            if ((flowControlTimes2++ % 1000) == 0) {
                log.warn("the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), pullRequest, flowControlTimes2);
            }
            return;
        }
    } else {
        if (processQueue.isLocked()) {
            if (!pullRequest.isLockedFirst()) {
                final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue());
                boolean brokerBusy = offset < pullRequest.getNextOffset();
                log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy);
                if (brokerBusy) {
                    log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", pullRequest, offset);
                }
                pullRequest.setLockedFirst(true);
                pullRequest.setNextOffset(offset);
            }
        } else {
            this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
            log.info("pull message later because not locked in broker, {}", pullRequest);
            return;
        }
    }
    final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
    if (null == subscriptionData) {
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        log.warn("find the consumer's subscription failed, {}", pullRequest);
        return;
    }
    final long beginTimestamp = System.currentTimeMillis();
    PullCallback pullCallback = new PullCallback() {

        @Override
        public void onSuccess(PullResult pullResult) {
            if (pullResult != null) {
                pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData);
                switch(pullResult.getPullStatus()) {
                    case FOUND:
                        long prevRequestOffset = pullRequest.getNextOffset();
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        long pullRT = System.currentTimeMillis() - beginTimestamp;
                        DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullRT);
                        long firstMsgOffset = Long.MAX_VALUE;
                        if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
                            DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        } else {
                            firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
                            DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
                            boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
                            // 
                            DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(// 
                            pullResult.getMsgFoundList(), // 
                            processQueue, // 
                            pullRequest.getMessageQueue(), dispathToConsume);
                            if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
                                // 间隔多久执行PullRequest
                                DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
                            } else {
                                DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                            }
                        }
                        if (// 
                        pullResult.getNextBeginOffset() < prevRequestOffset || firstMsgOffset < prevRequestOffset) {
                            log.warn(// 
                            "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", // 
                            pullResult.getNextBeginOffset(), // 
                            firstMsgOffset, prevRequestOffset);
                        }
                        break;
                    case NO_NEW_MSG:
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
                        DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        break;
                    case NO_MATCHED_MSG:
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
                        DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
                        break;
                    case OFFSET_ILLEGAL:
                        // 
                        log.warn(// 
                        "the pull request offset illegal, {} {}", pullRequest.toString(), pullResult.toString());
                        pullRequest.setNextOffset(pullResult.getNextBeginOffset());
                        pullRequest.getProcessQueue().setDropped(true);
                        DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() {

                            @Override
                            public void run() {
                                try {
                                    DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), false);
                                    DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
                                    DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
                                    log.warn("fix the pull request offset, {}", pullRequest);
                                } catch (Throwable e) {
                                    log.error("executeTaskLater Exception", e);
                                }
                            }
                        }, 10000);
                        break;
                    default:
                        break;
                }
            }
        }

        @Override
        public void onException(Throwable e) {
            if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                log.warn("execute the pull request exception", e);
            }
            DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
        }
    };
    boolean commitOffsetEnable = false;
    long commitOffsetValue = 0L;
    if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {
        commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);
        if (commitOffsetValue > 0) {
            commitOffsetEnable = true;
        }
    }
    String subExpression = null;
    boolean classFilter = false;
    SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
    if (sd != null) {
        if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) {
            subExpression = sd.getSubString();
        }
        classFilter = sd.isClassFilterMode();
    }
    int sysFlag = // 
    PullSysFlag.buildSysFlag(// commitOffset
    commitOffsetEnable, // suspend
    true, // subscription
    subExpression != null, // class filter
    classFilter);
    try {
        // 
        this.pullAPIWrapper.pullKernelImpl(// 1
        pullRequest.getMessageQueue(), // 2
        subExpression, // 3
        subscriptionData.getExpressionType(), // 4
        subscriptionData.getSubVersion(), // 5
        pullRequest.getNextOffset(), // 6
        this.defaultMQPushConsumer.getPullBatchSize(), // 7
        sysFlag, // 8
        commitOffsetValue, // 9
        BROKER_SUSPEND_MAX_TIME_MILLIS, // 10
        CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 11
        CommunicationMode.ASYNC, // 12
        pullCallback);
    } catch (Exception e) {
        log.error("pullKernelImpl exception", e);
        this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
    }
}