Examples with MessageQueue org.apache.rocketmq.common.message.MessageQueue used on opensource projects

Example 1 with MessageQueue

use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.

the class ConsumerServiceImpl method resetOffset.

@Override
@MultiMQAdminCmdMethod
public Map<String, ConsumerGroupRollBackStat> resetOffset(ResetOffsetRequest resetOffsetRequest) {
    Map<String, ConsumerGroupRollBackStat> groupRollbackStats = Maps.newHashMap();
    for (String consumerGroup : resetOffsetRequest.getConsumerGroupList()) {
        try {
            Map<MessageQueue, Long> rollbackStatsMap = mqAdminExt.resetOffsetByTimestamp(resetOffsetRequest.getTopic(), consumerGroup, resetOffsetRequest.getResetTime(), resetOffsetRequest.isForce());
            ConsumerGroupRollBackStat consumerGroupRollBackStat = new ConsumerGroupRollBackStat(true);
            List<RollbackStats> rollbackStatsList = consumerGroupRollBackStat.getRollbackStatsList();
            for (Map.Entry<MessageQueue, Long> rollbackStatsEntty : rollbackStatsMap.entrySet()) {
                RollbackStats rollbackStats = new RollbackStats();
                rollbackStats.setRollbackOffset(rollbackStatsEntty.getValue());
                rollbackStats.setQueueId(rollbackStatsEntty.getKey().getQueueId());
                rollbackStats.setBrokerName(rollbackStatsEntty.getKey().getBrokerName());
                rollbackStatsList.add(rollbackStats);
            }
            groupRollbackStats.put(consumerGroup, consumerGroupRollBackStat);
        } catch (MQClientException e) {
            if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) {
                try {
                    ConsumerGroupRollBackStat consumerGroupRollBackStat = new ConsumerGroupRollBackStat(true);
                    List<RollbackStats> rollbackStatsList = mqAdminExt.resetOffsetByTimestampOld(consumerGroup, resetOffsetRequest.getTopic(), resetOffsetRequest.getResetTime(), true);
                    consumerGroupRollBackStat.setRollbackStatsList(rollbackStatsList);
                    groupRollbackStats.put(consumerGroup, consumerGroupRollBackStat);
                    continue;
                } catch (Exception err) {
                    logger.error("op=resetOffset_which_not_online_error", err);
                }
            } else {
                logger.error("op=resetOffset_error", e);
            }
            groupRollbackStats.put(consumerGroup, new ConsumerGroupRollBackStat(false, e.getMessage()));
        } catch (Exception e) {
            logger.error("op=resetOffset_error", e);
            groupRollbackStats.put(consumerGroup, new ConsumerGroupRollBackStat(false, e.getMessage()));
        }
    }
    return groupRollbackStats;
}

Example 2 with MessageQueue

use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.

the class ConsumerServiceImpl method getClientConnection.

private Map<MessageQueue, String> getClientConnection(String groupName) {
    Map<MessageQueue, String> results = Maps.newHashMap();
    try {
        ConsumerConnection consumerConnection = mqAdminExt.examineConsumerConnectionInfo(groupName);
        for (Connection connection : consumerConnection.getConnectionSet()) {
            String clinetId = connection.getClientId();
            ConsumerRunningInfo consumerRunningInfo = mqAdminExt.getConsumerRunningInfo(groupName, clinetId, false);
            for (MessageQueue messageQueue : consumerRunningInfo.getMqTable().keySet()) {
                // results.put(messageQueue, clinetId + " " + connection.getClientAddr());
                results.put(messageQueue, clinetId);
            }
        }
    } catch (Exception err) {
        logger.error("op=getClientConnection_error", err);
    }
    return results;
}

Example 3 with MessageQueue

use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.

the class MessageServiceImpl method queryMessageByTopic.

@Override
public List<MessageView> queryMessageByTopic(String topic, final long begin, final long end) {
    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, null);
    List<MessageView> messageViewList = Lists.newArrayList();
    try {
        String subExpression = "*";
        consumer.start();
        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(topic);
        for (MessageQueue mq : mqs) {
            long minOffset = consumer.searchOffset(mq, begin);
            long maxOffset = consumer.searchOffset(mq, end);
            READQ: for (long offset = minOffset; offset <= maxOffset; ) {
                try {
                    if (messageViewList.size() > 2000) {
                        break;
                    }
                    PullResult pullResult = consumer.pull(mq, subExpression, offset, 32);
                    offset = pullResult.getNextBeginOffset();
                    switch(pullResult.getPullStatus()) {
                        case FOUND:
                            List<MessageView> messageViewListByQuery = Lists.transform(pullResult.getMsgFoundList(), new Function<MessageExt, MessageView>() {

                                @Override
                                public MessageView apply(MessageExt messageExt) {
                                    messageExt.setBody(null);
                                    return MessageView.fromMessageExt(messageExt);
                                }
                            });
                            List<MessageView> filteredList = Lists.newArrayList(Iterables.filter(messageViewListByQuery, new Predicate<MessageView>() {

                                @Override
                                public boolean apply(MessageView messageView) {
                                    if (messageView.getStoreTimestamp() < begin || messageView.getStoreTimestamp() > end) {
                                        logger.info("begin={} end={} time not in range {} {}", begin, end, messageView.getStoreTimestamp(), new Date(messageView.getStoreTimestamp()).toString());
                                    }
                                    return messageView.getStoreTimestamp() >= begin && messageView.getStoreTimestamp() <= end;
                                }
                            }));
                            messageViewList.addAll(filteredList);
                            break;
                        case NO_MATCHED_MSG:
                        case NO_NEW_MSG:
                        case OFFSET_ILLEGAL:
                            break READQ;
                    }
                } catch (Exception e) {
                    break;
                }
            }
        }
        Collections.sort(messageViewList, new Comparator<MessageView>() {

            @Override
            public int compare(MessageView o1, MessageView o2) {
                if (o1.getStoreTimestamp() - o2.getStoreTimestamp() == 0) {
                    return 0;
                }
                return (o1.getStoreTimestamp() > o2.getStoreTimestamp()) ? -1 : 1;
            }
        });
        return messageViewList;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    } finally {
        consumer.shutdown();
    }
}

Example 4 with MessageQueue

use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.

the class RocketMQSourceTest method testSource.

@Test
public void testSource() throws Exception {
    List<MessageExt> msgFoundList = new ArrayList<>();
    MessageExt messageExt = new MessageExt();
    messageExt.setKeys("keys");
    messageExt.setBody("body data".getBytes());
    messageExt.setBornTimestamp(System.currentTimeMillis());
    msgFoundList.add(messageExt);
    PullResult pullResult = new PullResult(PullStatus.FOUND, 3, 1, 5, msgFoundList);
    when(consumer.fetchConsumeOffset(any(MessageQueue.class), anyBoolean())).thenReturn(2L);
    when(consumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenReturn(pullResult);
    SourceContext context = mock(SourceContext.class);
    when(context.getCheckpointLock()).thenReturn(new Object());
    rocketMQSource.run(context);
    // schedule the pull task
    Set<MessageQueue> set = new HashSet();
    set.add(new MessageQueue(topic, "brk", 1));
    pullConsumerScheduleService.putTask(topic, set);
    MessageExt msg = pullResult.getMsgFoundList().get(0);
    // atLeastOnce: re-pulling immediately when messages found before
    verify(context, atLeastOnce()).collectWithTimestamp(deserializationSchema.deserializeKeyAndValue(msg.getKeys().getBytes(), msg.getBody()), msg.getBornTimestamp());
}

Example 5 with MessageQueue

use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.

the class RocketMQSinkTest method testBatchEvent.

@Test
public void testBatchEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
    /*
        start sink
         */
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    context.put(BATCH_SIZE_CONFIG, String.valueOf(batchSize));
    RocketMQSink sink = new RocketMQSink();
    Configurables.configure(sink, context);
    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();
    /*
        mock flume source
         */
    Map<String, String> msgs = new HashMap<>();
    Transaction tx = channel.getTransaction();
    tx.begin();
    int sendNum = 0;
    for (int i = 0; i < batchSize; i++) {
        String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss:SSSS");
        Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
        channel.put(event);
        log.info("publish message : {}", sendMsg);
        String[] sendMsgKv = sendMsg.split(",");
        msgs.put(sendMsgKv[1], sendMsgKv[0]);
        sendNum++;
        Thread.sleep(10);
    }
    log.info("send message num={}", sendNum);
    tx.commit();
    tx.close();
    Sink.Status status = sink.process();
    if (status == Sink.Status.BACKOFF) {
        fail("Error");
    }
    sink.stop();
    /*
        consumer message
         */
    consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setNamesrvAddr(nameServer);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
    consumer.start();
    int receiveNum = 0;
    String receiveMsg = null;
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
    for (MessageQueue queue : queues) {
        long offset = getMessageQueueOffset(queue);
        PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            for (MessageExt message : pullResult.getMsgFoundList()) {
                byte[] body = message.getBody();
                receiveMsg = new String(body, "UTF-8");
                String[] receiveMsgKv = receiveMsg.split(",");
                msgs.remove(receiveMsgKv[1]);
                log.info("receive message : {}", receiveMsg);
                receiveNum++;
            }
            long nextBeginOffset = pullResult.getNextBeginOffset();
            putMessageQueueOffset(queue, nextBeginOffset);
        }
    }
    log.info("receive message num={}", receiveNum);
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    consumer.shutdown();
    assertEquals(msgs.size(), 0);
}