Examples with Channel org.apache.flume.Channel used on opensource projects

Example 36 with Channel

use of org.apache.flume.Channel in project ignite by apache.

the class IgniteSinkTest method testSink.

/**
 * @throws Exception {@link Exception}.
 */
public void testSink() throws Exception {
    IgniteConfiguration cfg = loadConfiguration("modules/flume/src/test/resources/example-ignite.xml");
    cfg.setClientMode(false);
    final Ignite grid = startGrid("igniteServerNode", cfg);
    Context channelContext = new Context();
    channelContext.put("capacity", String.valueOf(EVENT_CNT));
    channelContext.put("transactionCapacity", String.valueOf(EVENT_CNT));
    Channel memoryChannel = new MemoryChannel();
    Configurables.configure(memoryChannel, channelContext);
    final CountDownLatch latch = new CountDownLatch(EVENT_CNT);
    final IgnitePredicate<Event> putLsnr = new IgnitePredicate<Event>() {

        @Override
        public boolean apply(Event evt) {
            assert evt != null;
            latch.countDown();
            return true;
        }
    };
    IgniteSink sink = new IgniteSink() {

        // Setting the listener on cache before sink processing starts.
        @Override
        public synchronized void start() {
            super.start();
            grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).localListen(putLsnr, EVT_CACHE_OBJECT_PUT);
        }
    };
    sink.setName("IgniteSink");
    sink.setChannel(memoryChannel);
    Context ctx = new Context();
    ctx.put(IgniteSinkConstants.CFG_CACHE_NAME, CACHE_NAME);
    ctx.put(IgniteSinkConstants.CFG_PATH, "example-ignite.xml");
    ctx.put(IgniteSinkConstants.CFG_EVENT_TRANSFORMER, "org.apache.ignite.stream.flume.TestEventTransformer");
    Configurables.configure(sink, ctx);
    sink.start();
    try {
        Transaction tx = memoryChannel.getTransaction();
        tx.begin();
        for (int i = 0; i < EVENT_CNT; i++) memoryChannel.put(EventBuilder.withBody((String.valueOf(i) + ": " + i).getBytes()));
        tx.commit();
        tx.close();
        Sink.Status status = Sink.Status.READY;
        while (status != Sink.Status.BACKOFF) {
            status = sink.process();
        }
    } finally {
        sink.stop();
    }
    // Checks that 10000 events successfully processed in 10 seconds.
    assertTrue(latch.await(10, TimeUnit.SECONDS));
    grid.events(grid.cluster().forCacheNodes(CACHE_NAME)).stopLocalListen(putLsnr);
    IgniteCache<String, Integer> cache = grid.cache(CACHE_NAME);
    // Checks that each event was processed properly.
    for (int i = 0; i < EVENT_CNT; i++) {
        assertEquals(i, (int) cache.get(String.valueOf(i)));
    }
    assertEquals(EVENT_CNT, cache.size(CachePeekMode.PRIMARY));
}

Example 37 with Channel

use of org.apache.flume.Channel in project rocketmq-externals by apache.

the class RocketMQSink method process.

@Override
public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction transaction = null;
    try {
        transaction = channel.getTransaction();
        transaction.begin();
        /*
            batch take
             */
        List<Event> events = new ArrayList<>();
        long beginTime = System.currentTimeMillis();
        while (true) {
            Event event = channel.take();
            if (event != null) {
                events.add(event);
            }
            if (events.size() == batchSize || System.currentTimeMillis() - beginTime > maxProcessTime) {
                break;
            }
        }
        if (events.size() == 0) {
            sinkCounter.incrementBatchEmptyCount();
            transaction.rollback();
            return Status.BACKOFF;
        }
        /*
            async send
             */
        CountDownLatch latch = new CountDownLatch(events.size());
        AtomicInteger errorNum = new AtomicInteger();
        for (Event event : events) {
            byte[] body = event.getBody();
            Message message = new Message(topic, tag, body);
            if (log.isDebugEnabled()) {
                log.debug("Processing event,body={}", new String(body, "UTF-8"));
            }
            producer.send(message, new SendCallBackHandler(message, latch, errorNum));
        }
        latch.await();
        sinkCounter.addToEventDrainAttemptCount(events.size());
        if (errorNum.get() > 0) {
            log.error("errorNum=" + errorNum + ",transaction will rollback");
            transaction.rollback();
            return Status.BACKOFF;
        } else {
            transaction.commit();
            sinkCounter.addToEventDrainSuccessCount(events.size());
            return Status.READY;
        }
    } catch (Exception e) {
        log.error("Failed to processing event", e);
        if (transaction != null) {
            try {
                transaction.rollback();
            } catch (Exception ex) {
                log.error("Failed to rollback transaction", ex);
                throw new EventDeliveryException("Failed to rollback transaction", ex);
            }
        }
        return Status.BACKOFF;
    } finally {
        if (transaction != null) {
            transaction.close();
        }
    }
}

Example 38 with Channel

use of org.apache.flume.Channel in project rocketmq-externals by apache.

the class RocketMQSourceTest method testEvent.

@Test
public void testEvent() throws EventDeliveryException, MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException {
    // publish test message
    DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
    producer.setNamesrvAddr(nameServer);
    String sendMsg = "\"Hello Flume\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-DD hh:mm:ss");
    try {
        producer.start();
        Message msg = new Message(TOPIC_DEFAULT, tag, sendMsg.getBytes("UTF-8"));
        SendResult sendResult = producer.send(msg);
        log.info("publish message : {}, sendResult:{}", sendMsg, sendResult);
    } catch (Exception e) {
        throw new MQClientException("Failed to publish messages", e);
    } finally {
        producer.shutdown();
    }
    // start source
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector channelSelector = new ReplicatingChannelSelector();
    channelSelector.setChannels(channels);
    ChannelProcessor channelProcessor = new ChannelProcessor(channelSelector);
    RocketMQSource source = new RocketMQSource();
    source.setChannelProcessor(channelProcessor);
    Configurables.configure(source, context);
    source.start();
    PollableSource.Status status = source.process();
    if (status == PollableSource.Status.BACKOFF) {
        fail("Error");
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    source.stop();
    /*
        mock flume sink
         */
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    if (event == null) {
        transaction.commit();
        fail("Error");
    }
    byte[] body = event.getBody();
    String receiveMsg = new String(body, "UTF-8");
    log.info("receive message : {}", receiveMsg);
    assertEquals(sendMsg, receiveMsg);
}