Examples with MessageExt org.apache.rocketmq.common.message.MessageExt used on opensource projects

Example 6 with MessageExt

use of org.apache.rocketmq.common.message.MessageExt in project rocketmq-externals by apache.

the class RocketMQSinkTest method testBatchEvent.

@Test
public void testBatchEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
    /*
        start sink
         */
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    context.put(BATCH_SIZE_CONFIG, String.valueOf(batchSize));
    RocketMQSink sink = new RocketMQSink();
    Configurables.configure(sink, context);
    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();
    /*
        mock flume source
         */
    Map<String, String> msgs = new HashMap<>();
    Transaction tx = channel.getTransaction();
    tx.begin();
    int sendNum = 0;
    for (int i = 0; i < batchSize; i++) {
        String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss:SSSS");
        Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
        channel.put(event);
        log.info("publish message : {}", sendMsg);
        String[] sendMsgKv = sendMsg.split(",");
        msgs.put(sendMsgKv[1], sendMsgKv[0]);
        sendNum++;
        Thread.sleep(10);
    }
    log.info("send message num={}", sendNum);
    tx.commit();
    tx.close();
    Sink.Status status = sink.process();
    if (status == Sink.Status.BACKOFF) {
        fail("Error");
    }
    sink.stop();
    /*
        consumer message
         */
    consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setNamesrvAddr(nameServer);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
    consumer.start();
    int receiveNum = 0;
    String receiveMsg = null;
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
    for (MessageQueue queue : queues) {
        long offset = getMessageQueueOffset(queue);
        PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            for (MessageExt message : pullResult.getMsgFoundList()) {
                byte[] body = message.getBody();
                receiveMsg = new String(body, "UTF-8");
                String[] receiveMsgKv = receiveMsg.split(",");
                msgs.remove(receiveMsgKv[1]);
                log.info("receive message : {}", receiveMsg);
                receiveNum++;
            }
            long nextBeginOffset = pullResult.getNextBeginOffset();
            putMessageQueueOffset(queue, nextBeginOffset);
        }
    }
    log.info("receive message num={}", receiveNum);
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    consumer.shutdown();
    assertEquals(msgs.size(), 0);
}

Example 7 with MessageExt

use of org.apache.rocketmq.common.message.MessageExt in project rocketmq-externals by apache.

the class RocketMQSource method doProcess.

@Override
protected Status doProcess() throws EventDeliveryException {
    List<Event> events = new ArrayList<>();
    Map<MessageQueue, Long> offsets = new HashMap<>();
    Event event;
    Map<String, String> headers;
    try {
        Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(topic);
        for (MessageQueue queue : queues) {
            long offset = getMessageQueueOffset(queue);
            PullResult pullResult = consumer.pull(queue, tag, offset, batchSize);
            if (log.isDebugEnabled()) {
                log.debug("Pull from queueId:{}, offset:{}, pullResult:{}", queue.getQueueId(), offset, pullResult);
            }
            if (pullResult.getPullStatus() == PullStatus.FOUND) {
                for (MessageExt msg : pullResult.getMsgFoundList()) {
                    byte[] body = msg.getBody();
                    headers = new HashMap<>();
                    headers.put(HEADER_TOPIC_NAME, topic);
                    headers.put(HEADER_TAG_NAME, tag);
                    if (log.isDebugEnabled()) {
                        log.debug("Processing message,body={}", new String(body, "UTF-8"));
                    }
                    event = EventBuilder.withBody(body, headers);
                    events.add(event);
                }
                offsets.put(queue, pullResult.getNextBeginOffset());
            }
        }
        if (events.size() > 0) {
            sourceCounter.incrementAppendBatchReceivedCount();
            sourceCounter.addToEventReceivedCount(events.size());
            getChannelProcessor().processEventBatch(events);
            sourceCounter.incrementAppendBatchAcceptedCount();
            sourceCounter.addToEventAcceptedCount(events.size());
            events.clear();
        }
        for (Map.Entry<MessageQueue, Long> entry : offsets.entrySet()) {
            putMessageQueueOffset(entry.getKey(), entry.getValue());
        }
    } catch (Exception e) {
        log.error("Failed to consumer message", e);
        return Status.BACKOFF;
    }
    return Status.READY;
}

Example 8 with MessageExt

use of org.apache.rocketmq.common.message.MessageExt in project rocketmq-externals by apache.

the class MessageConverter method convert2RMQMessage.

public static Message convert2RMQMessage(JmsBaseMessage jmsMsg) throws Exception {
    Message rocketmqMsg = new MessageExt();
    // 1. Transform message body
    rocketmqMsg.setBody(MessageConverter.getContentFromJms(jmsMsg));
    // 2. Transform topic and messageType
    JmsBaseTopic destination = (JmsBaseTopic) jmsMsg.getHeaders().get(JmsBaseConstant.JMS_DESTINATION);
    String topic = destination.getMessageTopic();
    rocketmqMsg.setTopic(topic);
    String messageType = destination.getMessageType();
    Preconditions.checkState(!messageType.contains("||"), "'||' can not be in the destination when sending a message");
    rocketmqMsg.setTags(messageType);
    // 3. Transform message properties
    Properties properties = initRocketMQHeaders(jmsMsg, topic, messageType);
    for (String name : properties.stringPropertyNames()) {
        String value = properties.getProperty(name);
        if (MessageConst.PROPERTY_KEYS.equals(name)) {
            rocketmqMsg.setKeys(value);
        } else if (MessageConst.PROPERTY_TAGS.equals(name)) {
            rocketmqMsg.setTags(value);
        } else if (MessageConst.PROPERTY_DELAY_TIME_LEVEL.equals(name)) {
            rocketmqMsg.setDelayTimeLevel(Integer.parseInt(value));
        } else if (MessageConst.PROPERTY_WAIT_STORE_MSG_OK.equals(name)) {
            rocketmqMsg.setWaitStoreMsgOK(Boolean.parseBoolean(value));
        } else if (MessageConst.PROPERTY_BUYER_ID.equals(name)) {
            rocketmqMsg.setBuyerId(value);
        } else {
            rocketmqMsg.putUserProperty(name, value);
        }
    }
    return rocketmqMsg;
}

Example 9 with MessageExt

use of org.apache.rocketmq.common.message.MessageExt in project rocketmq-externals by apache.

the class RocketMqUtilsTest method testConsumer.

@Test
public void testConsumer() throws MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    // start up spark
    Map<String, String> optionParams = new HashMap<>();
    optionParams.put(RocketMQConfig.NAME_SERVER_ADDR, NAME_SERVER);
    SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver").setMaster("local[*]");
    JavaStreamingContext sc = new JavaStreamingContext(sparkConf, new Duration(1000));
    List<String> topics = new ArrayList<>();
    topics.add(TOPIC_DEFAULT);
    LocationStrategy locationStrategy = LocationStrategy.PreferConsistent();
    JavaInputDStream<MessageExt> stream = RocketMqUtils.createJavaMQPullStream(sc, UUID.randomUUID().toString(), topics, ConsumerStrategy.earliest(), false, false, false, locationStrategy, optionParams);
    final Set<MessageExt> result = Collections.synchronizedSet(new HashSet<MessageExt>());
    stream.foreachRDD(new VoidFunction<JavaRDD<MessageExt>>() {

        @Override
        public void call(JavaRDD<MessageExt> messageExtJavaRDD) throws Exception {
            result.addAll(messageExtJavaRDD.collect());
        }
    });
    sc.start();
    long startTime = System.currentTimeMillis();
    boolean matches = false;
    while (!matches && System.currentTimeMillis() - startTime < 20000) {
        matches = MESSAGE_NUM == result.size();
        Thread.sleep(50);
    }
    sc.stop();
}

Example 10 with MessageExt

use of org.apache.rocketmq.common.message.MessageExt in project paascloud-master by paascloud.

the class MqConsumerStoreAspect method processMqConsumerStoreJoinPoint.

/**
 * Add exe time method object.
 *
 * @param joinPoint the join point
 *
 * @return the object
 *
 * @throws Throwable the throwable
 */
@Around(value = "mqConsumerStoreAnnotationPointcut()")
public Object processMqConsumerStoreJoinPoint(ProceedingJoinPoint joinPoint) throws Throwable {
    log.info("processMqConsumerStoreJoinPoint - 线程id={}", Thread.currentThread().getId());
    Object result;
    long startTime = System.currentTimeMillis();
    Object[] args = joinPoint.getArgs();
    MqConsumerStore annotation = getAnnotation(joinPoint);
    boolean isStorePreStatus = annotation.storePreStatus();
    List<MessageExt> messageExtList;
    if (args == null || args.length == 0) {
        throw new TpcBizException(ErrorCodeEnum.TPC10050005);
    }
    if (!(args[0] instanceof List)) {
        throw new TpcBizException(ErrorCodeEnum.GL99990001);
    }
    try {
        messageExtList = (List<MessageExt>) args[0];
    } catch (Exception e) {
        log.error("processMqConsumerStoreJoinPoint={}", e.getMessage(), e);
        throw new TpcBizException(ErrorCodeEnum.GL99990001);
    }
    MqMessageData dto = this.getTpcMqMessageDto(messageExtList.get(0));
    final String messageKey = dto.getMessageKey();
    if (isStorePreStatus) {
        mqMessageService.confirmReceiveMessage(consumerGroup, dto);
    }
    String methodName = joinPoint.getSignature().getName();
    try {
        result = joinPoint.proceed();
        log.info("result={}", result);
        if (CONSUME_SUCCESS.equals(result.toString())) {
            mqMessageService.saveAndConfirmFinishMessage(consumerGroup, messageKey);
        }
    } catch (Exception e) {
        log.error("发送可靠消息, 目标方法[{}], 出现异常={}", methodName, e.getMessage(), e);
        throw e;
    } finally {
        log.info("发送可靠消息 目标方法[{}], 总耗时={}", methodName, System.currentTimeMillis() - startTime);
    }
    return result;
}