Examples with DisruptorQueue backtype.storm.utils.DisruptorQueue used on opensource projects

Example 6 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class DisruptorTest method testMessageDisorder.

@Test
public void testMessageDisorder() throws InterruptedException {
    System.out.println("!!!!!!!!!!!!!!!!Begin testMessageDisorder!!!!!!!!!!");
    // Set queue length to bigger enough
    DisruptorQueue queue = createQueue("messageOrder", ProducerType.MULTI, 128);
    queue.publish("1");
    Runnable producer = new Producer(queue);
    final Object[] result = new Object[1];
    Runnable consumer = new Consumer(queue, new EventHandler<Object>() {

        private boolean head = true;

        private Map<String, Long> lastIdMap = new HashMap<String, Long>();

        @Override
        public void onEvent(Object obj, long sequence, boolean endOfBatch) throws Exception {
            consumerNum.incrementAndGet();
            if (head) {
                head = false;
                result[0] = obj;
            } else {
                String event = (String) obj;
                String[] item = event.split("@");
                Long current = Long.valueOf(item[1]);
                Long last = lastIdMap.get(item[0]);
                if (last != null) {
                    if (current <= last) {
                        String msg = "Consume disorder of " + item[0] + ", current" + current + ",last:" + last;
                        System.err.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
                        System.err.println(msg);
                        System.err.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
                        Assert.fail(msg);
                    }
                }
                lastIdMap.put(item[0], current);
            }
        }
    });
    run(producer, PRODUCER_NUM, 1000, consumer, 30000);
    Assert.assertEquals("We expect to receive first published message first, but received " + result[0], "1", result[0]);
    produceNum.incrementAndGet();
    Assert.assertEquals("produce: " + produceNum.get() + ", consume:" + consumerNum.get(), produceNum.get(), consumerNum.get());
    System.out.println("!!!!!!!!!!!!!!End testMessageDisorder!!!!!!!!!!!!");
}

Example 7 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class MkShuffer method isOutboundTaskAvailable.

private boolean isOutboundTaskAvailable(int taskId) {
    boolean ret = false;
    DisruptorQueue targetQueue = workerData.getInnerTaskTransfer().get(taskId);
    if (targetQueue != null) {
        float queueLoadRatio = targetQueue.pctFull();
        if (queueLoadRatio < loadMark) {
            ret = true;
        }
    } else {
        WorkerSlot slot = taskNodeport.get(taskId);
        if (slot != null) {
            IConnection connection = nodeportSocket.get(slot);
            if (connection != null) {
                ret = connection.available();
            }
        }
    }
    if (ret == false) {
        LOG.debug("taskId:{} is unavailable", taskId);
    }
    return ret;
}

Example 8 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class TaskReceiver method deserializer.

public boolean deserializer(KryoTupleDeserializer deserializer, boolean forceConsume) {
    //LOG.debug("start Deserializer of task, {}", taskId);
    boolean isIdling = true;
    DisruptorQueue exeQueue = innerTaskTransfer.get(taskId);
    if (!taskStatus.isShutdown()) {
        if ((deserializeQueue.population() > 0 && exeQueue.pctFull() < 1.0) || forceConsume) {
            try {
                List<Object> objects = deserializeQueue.retreiveAvailableBatch();
                for (Object object : objects) {
                    deserialize(deserializer, (byte[]) object, exeQueue);
                }
                isIdling = false;
            } catch (InterruptedException e) {
                LOG.error("InterruptedException " + e.getCause());
                return isIdling;
            } catch (TimeoutException e) {
                return isIdling;
            } catch (Throwable e) {
                if (Utils.exceptionCauseIsInstanceOf(KryoException.class, e)) {
                    throw new RuntimeException(e);
                } else if (!taskStatus.isShutdown()) {
                    LOG.error("Unknow exception ", e);
                }
            }
        }
    } else {
        task.unregisterDeserializeQueue();
    }
    return isIdling;
}

Example 9 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class TaskTransfer method transferControl.

public void transferControl(TupleExt tuple) {
    int taskId = tuple.getTargetTaskId();
    DisruptorQueue controlQueue = controlQueues.get(taskId);
    if (controlQueue != null) {
        controlQueue.publish(tuple);
    } else {
        transferControlQueue.publish(tuple);
    }
}

Example 10 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class BaseExecutors method registerInnerTransfer.

protected void registerInnerTransfer(DisruptorQueue disruptorQueue) {
    LOG.info("Registor inner transfer for executor thread of " + idStr);
    DisruptorQueue existInnerTransfer = innerTaskTransfer.get(taskId);
    if (existInnerTransfer != null) {
        LOG.info("Exist inner task transfer for executing thread of " + idStr);
        if (existInnerTransfer != disruptorQueue) {
            throw new RuntimeException("Inner task transfer must be only one in executing thread of " + idStr);
        }
    }
    innerTaskTransfer.put(taskId, disruptorQueue);
}