Examples with DisruptorQueue backtype.storm.utils.DisruptorQueue used on opensource projects

Example 11 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class Worker method startDispatchThread.

private AsyncLoopThread startDispatchThread() {
    // send tuple directly from netty server
    // send control tuple to dispatch thread
    // startDispatchDisruptor();
    IContext context = workerData.getContext();
    String topologyId = workerData.getTopologyId();
    //create recv connection
    Map stormConf = workerData.getStormConf();
    long timeout = JStormUtils.parseLong(stormConf.get(Config.TOPOLOGY_DISRUPTOR_WAIT_TIMEOUT), 10);
    WaitStrategy waitStrategy = new TimeoutBlockingWaitStrategy(timeout, TimeUnit.MILLISECONDS);
    int queueSize = JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_CTRL_BUFFER_SIZE), 256);
    DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI, queueSize, waitStrategy, false, 0, 0);
    //metric for recvControlQueue
    QueueGauge revCtrlGauge = new QueueGauge(recvControlQueue, MetricDef.RECV_CTRL_QUEUE);
    JStormMetrics.registerWorkerMetric(JStormMetrics.workerMetricName(MetricDef.RECV_CTRL_QUEUE, MetricType.GAUGE), new AsmGauge(revCtrlGauge));
    IConnection recvConnection = context.bind(topologyId, workerData.getPort(), workerData.getDeserializeQueues(), recvControlQueue, false, workerData.getTaskids());
    workerData.setRecvConnection(recvConnection);
    // create recvice control messages's thread
    RunnableCallback recvControlDispather = new VirtualPortCtrlDispatch(workerData, recvConnection, recvControlQueue, MetricDef.RECV_THREAD);
    return new AsyncLoopThread(recvControlDispather, false, Thread.MAX_PRIORITY, true);
}

Example 12 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class TaskTransfer method transfer.

public void transfer(TupleExt tuple) {
    int taskId = tuple.getTargetTaskId();
    DisruptorQueue exeQueue = innerTaskTransfer.get(taskId);
    DisruptorQueue targetQueue;
    if (exeQueue == null) {
        taskId = 0;
        targetQueue = serializeQueue;
    } else {
        targetQueue = exeQueue;
    }
    if (isBackpressureEnable) {
        Boolean backpressureStatus = targetTaskBackpressureStatus.get(taskId);
        if (backpressureStatus == null) {
            backpressureStatus = false;
            targetTaskBackpressureStatus.put(taskId, backpressureStatus);
        }
        if (backpressureStatus) {
            while (targetQueue.pctFull() > lowMark) {
                JStormUtils.sleepMs(1);
            }
            targetTaskBackpressureStatus.put(taskId, false);
            targetQueue.publish(tuple);
        } else {
            targetQueue.publish(tuple);
            if (targetQueue.pctFull() > highMark) {
                targetTaskBackpressureStatus.put(taskId, true);
            }
        }
    } else {
        targetQueue.publish(tuple);
    }
}

Example 13 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class MkLocalFirst method getActiveTask.

@Override
protected int getActiveTask(RandomRange randomrange, List<Integer> outTasks) {
    int index = randomrange.nextInt();
    int size = outTasks.size();
    int i = 0;
    for (i = 0; i < size; i++) {
        Integer taskId = outTasks.get(index);
        boolean taskStatus = workerData.isOutboundTaskActive(taskId);
        DisruptorQueue exeQueue = (workerData.getInnerTaskTransfer().get(taskId));
        float queueLoadRatio = exeQueue != null ? exeQueue.pctFull() : 0;
        if (taskStatus && queueLoadRatio < loadMark)
            break;
        else
            index = randomrange.nextInt();
    }
    return (i < size ? index : -1);
}

Example 14 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class NettyUnitTest method initNettyServer.

private IConnection initNettyServer(int port) {
    ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    //ConcurrentHashMap<Integer, DisruptorQueue> deserializeCtrlQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    WaitStrategy wait = (WaitStrategy) Utils.newInstance("com.lmax.disruptor.TimeoutBlockingWaitStrategy", 5, TimeUnit.MILLISECONDS);
    DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI, 256, wait, false, 0, 0);
    Set<Integer> taskSet = new HashSet<Integer>();
    taskSet.add(1);
    IConnection server = context.bind(null, port, deserializeQueues, recvControlQueue, true, taskSet);
    WaitStrategy waitStrategy = new BlockingWaitStrategy();
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy, false, 0, 0);
    server.registerQueue(task, recvQueue);
    return server;
}

Example 15 with DisruptorQueue

use of backtype.storm.utils.DisruptorQueue in project jstorm by alibaba.

the class DisruptorTest method testSingleProducer.

@Test
public void testSingleProducer() throws InterruptedException {
    System.out.println("!!!!!!!!!!!!!!Begin testSingleProducer!!!!!!!!!!!!!!");
    final AtomicBoolean messageConsumed = new AtomicBoolean(false);
    // Set queue length to 1, so that the RingBuffer can be easily full
    // to trigger consumer blocking
    DisruptorQueue queue = createQueue("consumerHang", ProducerType.SINGLE, 1);
    push(queue, 1);
    Runnable producer = new Producer(queue);
    Runnable consumer = new Consumer(queue, new EventHandler<Object>() {

        long count = 0;

        @Override
        public void onEvent(Object obj, long sequence, boolean endOfBatch) throws Exception {
            messageConsumed.set(true);
            System.out.println("Consume " + count++);
        }
    });
    run(producer, 0, 0, consumer, 50);
    Assert.assertTrue("disruptor message is never consumed due to consumer thread hangs", messageConsumed.get());
    System.out.println("!!!!!!!!!!!!!!End testSingleProducer!!!!!!!!!!!!!!");
}