Examples with MemoryChannel org.apache.flume.channel.MemoryChannel used on opensource projects

Example 11 with MemoryChannel

use of org.apache.flume.channel.MemoryChannel in project rocketmq-externals by apache.

the class RocketMQSinkTest method testEvent.

@Test
public void testEvent() throws MQClientException, InterruptedException, EventDeliveryException, RemotingException, MQBrokerException, UnsupportedEncodingException {
    /*
        start sink
         */
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    RocketMQSink sink = new RocketMQSink();
    Configurables.configure(sink, context);
    MemoryChannel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();
    /*
        mock flume source
         */
    String sendMsg = "\"Hello RocketMQ\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(sendMsg.getBytes(), null);
    channel.put(event);
    tx.commit();
    tx.close();
    log.info("publish message : {}", sendMsg);
    Sink.Status status = sink.process();
    if (status == Sink.Status.BACKOFF) {
        fail("Error");
    }
    sink.stop();
    /*
        consumer message
         */
    consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setNamesrvAddr(nameServer);
    consumer.setMessageModel(MessageModel.valueOf("BROADCASTING"));
    consumer.registerMessageQueueListener(TOPIC_DEFAULT, null);
    consumer.start();
    String receiveMsg = null;
    Set<MessageQueue> queues = consumer.fetchSubscribeMessageQueues(TOPIC_DEFAULT);
    for (MessageQueue queue : queues) {
        long offset = getMessageQueueOffset(queue);
        PullResult pullResult = consumer.pull(queue, tag, offset, 32);
        if (pullResult.getPullStatus() == PullStatus.FOUND) {
            for (MessageExt message : pullResult.getMsgFoundList()) {
                byte[] body = message.getBody();
                receiveMsg = new String(body, "UTF-8");
                log.info("receive message : {}", receiveMsg);
            }
            long nextBeginOffset = pullResult.getNextBeginOffset();
            putMessageQueueOffset(queue, nextBeginOffset);
        }
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    consumer.shutdown();
    assertEquals(sendMsg, receiveMsg);
}

Example 12 with MemoryChannel

use of org.apache.flume.channel.MemoryChannel in project rocketmq-externals by apache.

the class RocketMQSourceTest method testEvent.

@Test
public void testEvent() throws EventDeliveryException, MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    // publish test message
    DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
    producer.setNamesrvAddr(nameServer);
    String sendMsg = "\"Hello Flume\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    try {
        producer.start();
        Message msg = new Message(TOPIC_DEFAULT, tag, sendMsg.getBytes("UTF-8"));
        SendResult sendResult = producer.send(msg);
        log.info("publish message : {}, sendResult:{}", sendMsg, sendResult);
    } catch (Exception e) {
        throw new MQClientException("Failed to publish messages", e);
    } finally {
        producer.shutdown();
    }
    // start source
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector channelSelector = new ReplicatingChannelSelector();
    channelSelector.setChannels(channels);
    ChannelProcessor channelProcessor = new ChannelProcessor(channelSelector);
    RocketMQSource source = new RocketMQSource();
    source.setChannelProcessor(channelProcessor);
    Configurables.configure(source, context);
    source.start();
    PollableSource.Status status = source.process();
    if (status == PollableSource.Status.BACKOFF) {
        fail("Error");
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    source.stop();
    /*
        mock flume sink
         */
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    if (event == null) {
        transaction.commit();
        fail("Error");
    }
    byte[] body = event.getBody();
    String receiveMsg = new String(body, "UTF-8");
    log.info("receive message : {}", receiveMsg);
    assertEquals(sendMsg, receiveMsg);
}