Examples with DisruptorQueue backtype.storm.utils.DisruptorQueue used on opensource projects

Example 1 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class VirtualPortCtrlDispatch method handleEvent.

@Override
public void handleEvent(Object event, boolean endOfBatch) throws Exception {
    TaskMessage message = (TaskMessage) event;
    int task = message.task();
    Object tuple = null;
    try {
        //it maybe happened errors when update_topology
        tuple = deserialize(message.message(), task);
    } catch (Throwable e) {
        if (Utils.exceptionCauseIsInstanceOf(KryoException.class, e))
            throw new RuntimeException(e);
        LOG.warn("serialize happened errors!!! {}", e);
    }
    DisruptorQueue queue = controlQueues.get(task);
    if (queue == null) {
        LOG.warn("Received invalid control message form task-{}, Dropping...{} ", task, tuple);
        return;
    }
    if (tuple != null) {
        queue.publish(tuple);
    }
}

Example 2 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class NettyServer method enqueue.

/**
     * enqueue a received message
     * 
     * @param message
     * @throws InterruptedException
     */
public void enqueue(TaskMessage message, Channel channel) {
    //lots of messages may loss, when deserializeQueue haven't finish init operation
    while (!bstartRec) {
        LOG.info("check deserializeQueues have already been created");
        boolean isFinishInit = true;
        for (Integer task : workerTasks) {
            if (deserializeQueues.get(task) == null) {
                isFinishInit = false;
                JStormUtils.sleepMs(10);
                break;
            }
        }
        if (isFinishInit) {
            bstartRec = isFinishInit;
        }
    }
    short type = message.get_type();
    if (type == TaskMessage.NORMAL_MESSAGE) {
        //enqueue a received message
        int task = message.task();
        DisruptorQueue queue = deserializeQueues.get(task);
        if (queue == null) {
            LOG.warn("Received invalid message directed at task {}. Dropping...", task);
            LOG.debug("Message data: {}", JStormUtils.toPrintableString(message.message()));
            return;
        }
        if (!isBackpressureEnable) {
            queue.publish(message.message());
        } else {
            String remoteAddress = channel.getRemoteAddress().toString();
            flowCtrl(channel, remoteAddress, queue, task, message.message());
        }
    } else if (type == TaskMessage.CONTROL_MESSAGE) {
        //enqueue a control message
        if (recvControlQueue == null) {
            LOG.info("Can not find the recvControlQueue. So, dropping this control message");
            return;
        }
        recvControlQueue.publish(message);
    } else {
        LOG.warn("Unexpected message (type={}) was received from task {}", type, message.task());
    }
}

Example 3 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class DisruptorTest method testConsumeBatchWhenAvailable.

@Test
public void testConsumeBatchWhenAvailable() {
    System.out.println("!!!!!!!!!!!!!!!Begin testConsumeBatchWhenAvailable!!!!!!!!!!!!");
    resetNum();
    // Set queue length to bigger enough
    DisruptorQueue queue = createQueue("messageOrder", ProducerType.MULTI, 128);
    push(queue, 128);
    queue.consumeBatchWhenAvailable(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testConsumeBatchWhenAvailable test 1");
    resetNum();
    //queue.consumerStarted();
    push(queue, 128);
    queue.consumeBatchWhenAvailable(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testConsumeBatchWhenAvailable test 2");
    System.out.println("!!!!!!!!!!!!!!!Finsh testConsumeBatchWhenAvailable for MULTI!!!!!!!!!!!!");
    resetNum();
    // Set queue length to bigger enough
    DisruptorQueue queue2 = createQueue("messageOrder", ProducerType.SINGLE, 128);
    push(queue2, 128);
    queue2.consumeBatchWhenAvailable(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testConsumeBatchWhenAvailable test 3");
    resetNum();
    //queue2.consumerStarted();
    push(queue2, 128);
    queue2.consumeBatchWhenAvailable(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testConsumeBatchWhenAvailable test 4");
    System.out.println("!!!!!!!!!!!!!!!Finsh testConsumeBatchWhenAvailable for single !!!!!!!!!!!!");
    System.out.println("!!!!!!!!!!!!!End testConsumeBatchWhenAvailable!!!!!!!!!!!");
}

Example 4 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class DisruptorTest method testTryConsume.

@Test
public void testTryConsume() {
    System.out.println("!!!!!!!!!!!!Begin testTryConsume!!!!!!!!!!!!!!!!");
    resetNum();
    // Set queue length to bigger enough
    DisruptorQueue queue = createQueue("messageOrder", ProducerType.MULTI, 128);
    push(queue, 128);
    queue.consumeBatch(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testTryConsume test 1");
    resetNum();
    //queue.consumerStarted();
    push(queue, 128);
    queue.consumeBatch(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testTryConsume test 2");
    resetNum();
    // Set queue length to bigger enough
    DisruptorQueue queue2 = createQueue("messageOrder", ProducerType.SINGLE, 128);
    push(queue2, 128);
    queue2.consumeBatch(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testTryConsume test 3");
    resetNum();
    //queue2.consumerStarted();
    push(queue2, 128);
    queue2.consumeBatch(handler);
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!! finish testTryConsume test 4");
    System.out.println("!!!!!!!!!!!!!!!!!End testTryConsume!!!!!!!!!!!!!!");
}

Example 5 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class DisruptorTest method testBeforeStartConsumer.

// Remove the cache in disruptor queue. So comment out this tc.
/*    @Test
    public void testLaterStartConsumer() throws InterruptedException {
        System.out.println("!!!!!!!!!!!!!!!Begin testLaterStartConsumer!!!!!!!!!!");
        final AtomicBoolean messageConsumed = new AtomicBoolean(false);

        // Set queue length to 1, so that the RingBuffer can be easily full
        // to trigger consumer blocking
        DisruptorQueue queue = createQueue("consumerHang", ProducerType.MULTI, 2);
        push(queue, 1);
        Runnable producer = new Producer(queue);
        Runnable consumer = new Consumer(queue, new EventHandler<Object>() {
            long count = 0;

            @Override
            public void onEvent(Object obj, long sequence, boolean endOfBatch) throws Exception {

                messageConsumed.set(true);
                System.out.println("Consume " + count++);
            }
        });

        run(producer, 0, 0, consumer, 50);
        Assert.assertTrue("disruptor message is never consumed due to consumer thread hangs", messageConsumed.get());

        System.out.println("!!!!!!!!!!!!!!!!End testLaterStartConsumer!!!!!!!!!!");
    }*/
@Test
public void testBeforeStartConsumer() throws InterruptedException {
    System.out.println("!!!!!!!!!!!!Begin testBeforeStartConsumer!!!!!!!!!");
    final AtomicBoolean messageConsumed = new AtomicBoolean(false);
    // Set queue length to 1, so that the RingBuffer can be easily full
    // to trigger consumer blocking
    DisruptorQueue queue = createQueue("consumerHang", ProducerType.MULTI, 2);
    //queue.consumerStarted();
    push(queue, 1);
    Runnable producer = new Producer(queue);
    Runnable consumer = new Consumer(queue, new EventHandler<Object>() {

        long count = 0;

        @Override
        public void onEvent(Object obj, long sequence, boolean endOfBatch) throws Exception {
            messageConsumed.set(true);
            System.out.println("Consume " + count++);
        }
    });
    run(producer, 0, 0, consumer, 50);
    Assert.assertTrue("disruptor message is never consumed due to consumer thread hangs", messageConsumed.get());
    System.out.println("!!!!!!!!!!!!!End testBeforeStartConsumer!!!!!!!!!!");
}