use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.
the class ConsumerServiceImpl method resetOffset.
@Override
@MultiMQAdminCmdMethod
public Map<String, ConsumerGroupRollBackStat> resetOffset(ResetOffsetRequest resetOffsetRequest) {
Map<String, ConsumerGroupRollBackStat> groupRollbackStats = Maps.newHashMap();
for (String consumerGroup : resetOffsetRequest.getConsumerGroupList()) {
try {
Map<MessageQueue, Long> rollbackStatsMap = mqAdminExt.resetOffsetByTimestamp(resetOffsetRequest.getTopic(), consumerGroup, resetOffsetRequest.getResetTime(), resetOffsetRequest.isForce());
ConsumerGroupRollBackStat consumerGroupRollBackStat = new ConsumerGroupRollBackStat(true);
List<RollbackStats> rollbackStatsList = consumerGroupRollBackStat.getRollbackStatsList();
for (Map.Entry<MessageQueue, Long> rollbackStatsEntty : rollbackStatsMap.entrySet()) {
RollbackStats rollbackStats = new RollbackStats();
rollbackStats.setRollbackOffset(rollbackStatsEntty.getValue());
rollbackStats.setQueueId(rollbackStatsEntty.getKey().getQueueId());
rollbackStats.setBrokerName(rollbackStatsEntty.getKey().getBrokerName());
rollbackStatsList.add(rollbackStats);
}
groupRollbackStats.put(consumerGroup, consumerGroupRollBackStat);
} catch (MQClientException e) {
if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) {
try {
ConsumerGroupRollBackStat consumerGroupRollBackStat = new ConsumerGroupRollBackStat(true);
List<RollbackStats> rollbackStatsList = mqAdminExt.resetOffsetByTimestampOld(consumerGroup, resetOffsetRequest.getTopic(), resetOffsetRequest.getResetTime(), true);
consumerGroupRollBackStat.setRollbackStatsList(rollbackStatsList);
groupRollbackStats.put(consumerGroup, consumerGroupRollBackStat);
continue;
} catch (Exception err) {
logger.error("op=resetOffset_which_not_online_error", err);
}
} else {
logger.error("op=resetOffset_error", e);
}
groupRollbackStats.put(consumerGroup, new ConsumerGroupRollBackStat(false, e.getMessage()));
} catch (Exception e) {
logger.error("op=resetOffset_error", e);
groupRollbackStats.put(consumerGroup, new ConsumerGroupRollBackStat(false, e.getMessage()));
}
}
return groupRollbackStats;
}
use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.
the class ConsumerServiceImpl method getClientConnection.
private Map<MessageQueue, String> getClientConnection(String groupName) {
Map<MessageQueue, String> results = Maps.newHashMap();
try {
ConsumerConnection consumerConnection = mqAdminExt.examineConsumerConnectionInfo(groupName);
for (Connection connection : consumerConnection.getConnectionSet()) {
String clinetId = connection.getClientId();
ConsumerRunningInfo consumerRunningInfo = mqAdminExt.getConsumerRunningInfo(groupName, clinetId, false);
for (MessageQueue messageQueue : consumerRunningInfo.getMqTable().keySet()) {
// results.put(messageQueue, clinetId + " " + connection.getClientAddr());
results.put(messageQueue, clinetId);
}
}
} catch (Exception err) {
logger.error("op=getClientConnection_error", err);
}
return results;
}
use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.
the class MessageServiceImpl method queryMessageByTopic.
@Override
public List<MessageView> queryMessageByTopic(String topic, final long begin, final long end) {
DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, null);
List<MessageView> messageViewList = Lists.newArrayList();
try {
String subExpression = "*";
consumer.start();
Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(topic);
for (MessageQueue mq : mqs) {
long minOffset = consumer.searchOffset(mq, begin);
long maxOffset = consumer.searchOffset(mq, end);
READQ: for (long offset = minOffset; offset <= maxOffset; ) {
try {
if (messageViewList.size() > 2000) {
break;
}
PullResult pullResult = consumer.pull(mq, subExpression, offset, 32);
offset = pullResult.getNextBeginOffset();
switch(pullResult.getPullStatus()) {
case FOUND:
List<MessageView> messageViewListByQuery = Lists.transform(pullResult.getMsgFoundList(), new Function<MessageExt, MessageView>() {
@Override
public MessageView apply(MessageExt messageExt) {
messageExt.setBody(null);
return MessageView.fromMessageExt(messageExt);
}
});
List<MessageView> filteredList = Lists.newArrayList(Iterables.filter(messageViewListByQuery, new Predicate<MessageView>() {
@Override
public boolean apply(MessageView messageView) {
if (messageView.getStoreTimestamp() < begin || messageView.getStoreTimestamp() > end) {
logger.info("begin={} end={} time not in range {} {}", begin, end, messageView.getStoreTimestamp(), new Date(messageView.getStoreTimestamp()).toString());
}
return messageView.getStoreTimestamp() >= begin && messageView.getStoreTimestamp() <= end;
}
}));
messageViewList.addAll(filteredList);
break;
case NO_MATCHED_MSG:
case NO_NEW_MSG:
case OFFSET_ILLEGAL:
break READQ;
}
} catch (Exception e) {
break;
}
}
}
Collections.sort(messageViewList, new Comparator<MessageView>() {
@Override
public int compare(MessageView o1, MessageView o2) {
if (o1.getStoreTimestamp() - o2.getStoreTimestamp() == 0) {
return 0;
}
return (o1.getStoreTimestamp() > o2.getStoreTimestamp()) ? -1 : 1;
}
});
return messageViewList;
} catch (Exception e) {
throw Throwables.propagate(e);
} finally {
consumer.shutdown();
}
}
use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.
the class RocketMQSourceTest method testSource.
@Test
public void testSource() throws Exception {
List<MessageExt> msgFoundList = new ArrayList<>();
MessageExt messageExt = new MessageExt();
messageExt.setKeys("keys");
messageExt.setBody("body data".getBytes());
messageExt.setBornTimestamp(System.currentTimeMillis());
msgFoundList.add(messageExt);
PullResult pullResult = new PullResult(PullStatus.FOUND, 3, 1, 5, msgFoundList);
when(consumer.fetchConsumeOffset(any(MessageQueue.class), anyBoolean())).thenReturn(2L);
when(consumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenReturn(pullResult);
SourceContext context = mock(SourceContext.class);
when(context.getCheckpointLock()).thenReturn(new Object());
rocketMQSource.run(context);
// schedule the pull task
Set<MessageQueue> set = new HashSet();
set.add(new MessageQueue(topic, "brk", 1));
pullConsumerScheduleService.putTask(topic, set);
MessageExt msg = pullResult.getMsgFoundList().get(0);
// atLeastOnce: re-pulling immediately when messages found before
verify(context, atLeastOnce()).collectWithTimestamp(deserializationSchema.deserializeKeyAndValue(msg.getKeys().getBytes(), msg.getBody()), msg.getBornTimestamp());
}
use of org.apache.rocketmq.common.message.MessageQueue in project rocketmq-externals by apache.
the class RocketMQSinkTest method testBatchEvent.
@Test
public void testBatchEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
/*
start sink
*/
Context context = new Context();
context.put(NAME_SERVER_CONFIG, nameServer);
context.put(TAG_CONFIG, tag);
context.put(BATCH_SIZE_CONFIG, String.valueOf(batchSize));
RocketMQSink sink = new RocketMQSink();
Configurables.configure(sink, context);
MemoryChannel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
/*
mock flume source
*/
Map<String, String> msgs = new HashMap<>();
Transaction tx = channel.getTransaction();
tx.begin();
int sendNum = 0;
for (int i = 0; i < batchSize; i++) {
String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss:SSSS");
Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
channel.put(event);
log.info("publish message : {}", sendMsg);
String[] sendMsgKv = sendMsg.split(",");
msgs.put(sendMsgKv[1], sendMsgKv[0]);
sendNum++;
Thread.sleep(10);
}
log.info("send message num={}", sendNum);
tx.commit();
tx.close();
Sink.Status status = sink.process();
if (status == Sink.Status.BACKOFF) {
fail("Error");
}
sink.stop();
/*
consumer message
*/
consumer = new DefaultMQPullConsumer(consumerGroup);
consumer.setNamesrvAddr(nameServer);
consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
consumer.start();
int receiveNum = 0;
String receiveMsg = null;
Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
for (MessageQueue queue : queues) {
long offset = getMessageQueueOffset(queue);
PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
if (pullResult.getPullStatus() == PullStatus.FOUND) {
for (MessageExt message : pullResult.getMsgFoundList()) {
byte[] body = message.getBody();
receiveMsg = new String(body, "UTF-8");
String[] receiveMsgKv = receiveMsg.split(",");
msgs.remove(receiveMsgKv[1]);
log.info("receive message : {}", receiveMsg);
receiveNum++;
}
long nextBeginOffset = pullResult.getNextBeginOffset();
putMessageQueueOffset(queue, nextBeginOffset);
}
}
log.info("receive message num={}", receiveNum);
/*
wait for processQueueTable init
*/
Thread.sleep(1000);
consumer.shutdown();
assertEquals(msgs.size(), 0);
}
Aggregations