Examples with PullResult org.apache.rocketmq.client.consumer.PullResult used on opensource projects

Example 21 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class RocketMQSource method run.

@Override
public void run(SourceContext context) throws Exception {
    LOG.debug("source run....");
    // The lock that guarantees that record emission and state updates are atomic,
    // from the view of taking a checkpoint.
    final Object lock = context.getCheckpointLock();
    int delayWhenMessageNotFound = getInteger(props, RocketMQConfig.CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND, RocketMQConfig.DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND);
    String tag = props.getProperty(RocketMQConfig.CONSUMER_TAG, RocketMQConfig.DEFAULT_CONSUMER_TAG);
    int pullPoolSize = getInteger(props, RocketMQConfig.CONSUMER_PULL_POOL_SIZE, RocketMQConfig.DEFAULT_CONSUMER_PULL_POOL_SIZE);
    int pullBatchSize = getInteger(props, RocketMQConfig.CONSUMER_BATCH_SIZE, RocketMQConfig.DEFAULT_CONSUMER_BATCH_SIZE);
    pullConsumerScheduleService.setPullThreadNums(pullPoolSize);
    pullConsumerScheduleService.registerPullTaskCallback(topic, new PullTaskCallback() {

        @Override
        public void doPullTask(MessageQueue mq, PullTaskContext pullTaskContext) {
            try {
                long offset = getMessageQueueOffset(mq);
                if (offset < 0) {
                    return;
                }
                PullResult pullResult = consumer.pull(mq, tag, offset, pullBatchSize);
                boolean found = false;
                switch(pullResult.getPullStatus()) {
                    case FOUND:
                        List<MessageExt> messages = pullResult.getMsgFoundList();
                        for (MessageExt msg : messages) {
                            byte[] key = msg.getKeys() != null ? msg.getKeys().getBytes(StandardCharsets.UTF_8) : null;
                            byte[] value = msg.getBody();
                            OUT data = schema.deserializeKeyAndValue(key, value);
                            // output and state update are atomic
                            synchronized (lock) {
                                context.collectWithTimestamp(data, msg.getBornTimestamp());
                            }
                        }
                        found = true;
                        break;
                    case NO_MATCHED_MSG:
                        LOG.debug("No matched message after offset {} for queue {}", offset, mq);
                        break;
                    case NO_NEW_MSG:
                        break;
                    case OFFSET_ILLEGAL:
                        LOG.warn("Offset {} is illegal for queue {}", offset, mq);
                        break;
                    default:
                        break;
                }
                synchronized (lock) {
                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());
                }
                if (found) {
                    // no delay when messages were found
                    pullTaskContext.setPullNextDelayTimeMillis(0);
                } else {
                    pullTaskContext.setPullNextDelayTimeMillis(delayWhenMessageNotFound);
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    });
    try {
        pullConsumerScheduleService.start();
    } catch (MQClientException e) {
        throw new RuntimeException(e);
    }
    runningChecker.setRunning(true);
    awaitTermination();
}

Example 22 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class ReplicatorTest method testCustomReplicationEndpoint.

/**
 * This method tests the replicator by writing data from hbase to rocketmq and reading it back.
 *
 * @throws IOException
 * @throws ReplicationException
 * @throws InterruptedException
 * @throws MQClientException
 * @throws RemotingException
 * @throws MQBrokerException
 */
@Test
public void testCustomReplicationEndpoint() throws IOException, ReplicationException, InterruptedException, MQClientException, RemotingException, MQBrokerException {
    final DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(CONSUMER_GROUP_NAME);
    try {
        createTestTable();
        final Map<TableName, List<String>> tableCfs = new HashMap<>();
        List<String> cfs = new ArrayList<>();
        cfs.add(COLUMN_FAMILY);
        tableCfs.put(TABLE_NAME, cfs);
        addPeer(utility.getConfiguration(), PEER_NAME, tableCfs);
        // wait for new peer to be added
        Thread.sleep(500);
        final int numberOfRecords = 10;
        final Transaction inTransaction = insertData(numberOfRecords);
        // wait for data to be replicated
        Thread.sleep(500);
        consumer.setNamesrvAddr(NAMESERVER);
        consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
        consumer.registerMessageQueueListener(ROCKETMQ_TOPIC, null);
        consumer.start();
        int receiveNum = 0;
        String receiveMsg = null;
        Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(ROCKETMQ_TOPIC);
        for (MessageQueue queue : queues) {
            long offset = getMessageQueueOffset(consumer, queue);
            PullResult pullResult = consumer.pull(queue, null, offset, batchSize);
            if (pullResult.getPullStatus() == PullStatus.FOUND) {
                for (MessageExt message : pullResult.getMsgFoundList()) {
                    byte[] body = message.getBody();
                    receiveMsg = new String(body, "UTF-8");
                    // String[] receiveMsgKv = receiveMsg.split(",");
                    // msgs.remove(receiveMsgKv[1]);
                    logger.info("receive message : {}", receiveMsg);
                    receiveNum++;
                }
                long nextBeginOffset = pullResult.getNextBeginOffset();
                consumer.updateConsumeOffset(queue, offset);
            }
        }
        logger.info("receive message num={}", receiveNum);
        // wait for processQueueTable init
        Thread.sleep(1000);
        assertEquals(inTransaction.toJson(), receiveMsg);
    } finally {
        removePeer();
        consumer.shutdown();
    }
}

Example 23 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class RocketMQConsumer method pull.

/**
 * Pulls messages from specified topics.
 *
 * @return a map containing a list of messages per topic
 * @throws MQClientException
 * @throws RemotingException
 * @throws InterruptedException
 * @throws MQBrokerException
 */
public Map<String, List<MessageExt>> pull() throws MQClientException, RemotingException, InterruptedException, MQBrokerException {
    final Map<String, List<MessageExt>> messagesPerTopic = new HashMap<>();
    for (String topic : topics) {
        final Set<MessageQueue> msgQueues = consumer.fetchSubscribeMessageQueues(topic);
        for (MessageQueue msgQueue : msgQueues) {
            final long offset = getMessageQueueOffset(msgQueue);
            final PullResult pullResult = consumer.pull(msgQueue, null, offset, batchSize);
            if (pullResult.getPullStatus() == PullStatus.FOUND) {
                messagesPerTopic.put(topic, pullResult.getMsgFoundList());
                logger.debug("Pulled {} messages", pullResult.getMsgFoundList().size());
            }
        }
    }
    return messagesPerTopic.size() > 0 ? messagesPerTopic : null;
}

Example 24 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class RocketMQSinkTest method testEvent.

@Test
public void testEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
    /*
        start sink
         */
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    RocketMQSink sink = new RocketMQSink();
    Configurables.configure(sink, context);
    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();
    /*
        mock flume source
         */
    String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
    channel.put(event);
    tx.commit();
    tx.close();
    log.info("publish message : {}", sendMsg);
    Sink.Status status = sink.process();
    if (status == Sink.Status.BACKOFF) {
        fail("Error");
    }
    sink.stop();
    /*
        consumer message
         */
    consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setNamesrvAddr(nameServer);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
    consumer.start();
    String receiveMsg = null;
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
    for (MessageQueue queue : queues) {
        long offset = getMessageQueueOffset(queue);
        PullResult pullResult = consumer.pull(queue, tag, offset, 32);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            for (MessageExt message : pullResult.getMsgFoundList()) {
                byte[] body = message.getBody();
                receiveMsg = new String(body, "UTF-8");
                log.info("receive message : {}", receiveMsg);
            }
            long nextBeginOffset = pullResult.getNextBeginOffset();
            putMessageQueueOffset(queue, nextBeginOffset);
        }
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    consumer.shutdown();
    assertEquals(sendMsg, receiveMsg);
}

Example 25 with PullResult

use of org.apache.rocketmq.client.consumer.PullResult in project rocketmq-externals by apache.

the class BinlogPositionManager method initPositionFromMqTail.

private void initPositionFromMqTail() throws Exception {
    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("BINLOG_CONSUMER_GROUP");
    consumer.setNamesrvAddr(config.mqNamesrvAddr);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.start();
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(config.mqTopic);
    MessageQueue queue = queues.iterator().next();
    if (queue != null) {
        Long offset = consumer.maxOffset(queue);
        if (offset > 0)
            offset--;
        PullResult pullResult = consumer.pull(queue, "*", offset, 100);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            MessageExt msg = pullResult.getMsgFoundList().get(0);
            String json = new String(msg.getBody(), "UTF-8");
            JSONObject js = JSON.parseObject(json);
            binlogFilename = (String) js.get("binlogFilename");
            nextPosition = js.getLong("nextPosition");
        }
    }
}