Examples with JCQueue org.apache.storm.utils.JCQueue used on opensource projects

Example 11 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testRefreshBackpressureWithEmptyOverflow.

@Test
public void testRefreshBackpressureWithEmptyOverflow() {
    int taskId = 1;
    JCQueue queue = mock(JCQueue.class);
    when(queue.isEmptyOverflow()).thenReturn(true);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of(taskId, queue), new StormMetricRegistry(), ImmutableMap.of(taskId, "component-1"));
    BackpressureState state = tracker.getBackpressureState(taskId);
    tracker.recordBackPressure(state);
    boolean backpressureChanged = tracker.refreshBpTaskList();
    BackPressureStatus status = tracker.getCurrStatus();
    assertThat(backpressureChanged, is(true));
    assertThat(status.workerId, is(WORKER_ID));
    assertThat(status.nonBpTasks, contains(taskId));
}

Example 12 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testRefreshBackPressureWithNonEmptyOverflow.

@Test
public void testRefreshBackPressureWithNonEmptyOverflow() {
    int taskId = 1;
    JCQueue queue = mock(JCQueue.class);
    when(queue.isEmptyOverflow()).thenReturn(false);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of(taskId, queue), new StormMetricRegistry(), ImmutableMap.of(taskId, "component-1"));
    BackpressureState state = tracker.getBackpressureState(taskId);
    tracker.recordBackPressure(state);
    boolean backpressureChanged = tracker.refreshBpTaskList();
    BackPressureStatus status = tracker.getCurrStatus();
    assertThat(backpressureChanged, is(false));
    assertThat(status.workerId, is(WORKER_ID));
    assertThat(status.bpTasks, contains(taskId));
}

Example 13 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testSetBackpressure.

@Test
public void testSetBackpressure() {
    int taskIdNoBackPressure = 1;
    JCQueue noBackPressureQueue = mock(JCQueue.class);
    int taskIdBackPressure = 2;
    JCQueue backPressureQueue = mock(JCQueue.class);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of(taskIdNoBackPressure, noBackPressureQueue, taskIdBackPressure, backPressureQueue), new StormMetricRegistry(), ImmutableMap.of(taskIdNoBackPressure, "NoBackPressureComponent", taskIdBackPressure, "BackPressureComponent"));
    BackpressureState state = tracker.getBackpressureState(taskIdBackPressure);
    boolean backpressureChanged = tracker.recordBackPressure(state);
    BackPressureStatus status = tracker.getCurrStatus();
    assertThat(backpressureChanged, is(true));
    assertThat(status.workerId, is(WORKER_ID));
    assertThat(status.nonBpTasks, contains(taskIdNoBackPressure));
    assertThat(status.bpTasks, contains(taskIdBackPressure));
}

Example 14 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class JCQueuePerfTest method oneProducer2Consumers.

private static void oneProducer2Consumers(int prodBatchSz) {
    WaitStrategyPark ws = new WaitStrategyPark(100);
    StormMetricRegistry registry = new StormMetricRegistry();
    JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
    JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
    final Producer2 prod1 = new Producer2(q1, q2);
    final Consumer cons1 = new Consumer(q1);
    final Consumer cons2 = new Consumer(q2);
    runAllThds(prod1, cons1, cons2);
}

Example 15 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class WorkerState method transferLocalBatch.

// Receives msgs from remote workers and feeds them to local executors. If any receiving local executor is under Back Pressure,
// informs other workers about back pressure situation. Runs in the NettyWorker thread.
private void transferLocalBatch(ArrayList<AddressedTuple> tupleBatch) {
    for (int i = 0; i < tupleBatch.size(); i++) {
        AddressedTuple tuple = tupleBatch.get(i);
        JCQueue queue = taskToExecutorQueue.get(tuple.dest);
        // 1- try adding to main queue if its overflow is not empty
        if (queue.isEmptyOverflow()) {
            if (queue.tryPublish(tuple)) {
                continue;
            }
        }
        // 2- BP detected (i.e MainQ is full). So try adding to overflow
        int currOverflowCount = queue.getOverflowCount();
        // get BP state object so only have to lookup once
        BackpressureState bpState = bpTracker.getBackpressureState(tuple.dest);
        if (bpTracker.recordBackPressure(bpState)) {
            receiver.sendBackPressureStatus(bpTracker.getCurrStatus());
            bpTracker.setLastOverflowCount(bpState, currOverflowCount);
        } else {
            if (currOverflowCount - bpTracker.getLastOverflowCount(bpState) > RESEND_BACKPRESSURE_SIZE) {
                // resend BP status, in case prev notification was missed or reordered
                BackPressureStatus bpStatus = bpTracker.getCurrStatus();
                receiver.sendBackPressureStatus(bpStatus);
                bpTracker.setLastOverflowCount(bpState, currOverflowCount);
                LOG.debug("Re-sent BackPressure Status. OverflowCount = {}, BP Status ID = {}. ", currOverflowCount, bpStatus.id);
            }
        }
        if (!queue.tryPublishToOverflow(tuple)) {
            dropMessage(tuple, queue);
        }
    }
}