Examples with JCQueue org.apache.storm.utils.JCQueue used on opensource projects

Example 6 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class WorkerState method mkReceiveQueueMap.

private Map<List<Long>, JCQueue> mkReceiveQueueMap(Map<String, Object> topologyConf, Set<List<Long>> executors, Map<Integer, String> taskToComponent) {
    Integer recvQueueSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE));
    Integer recvBatchSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE));
    Integer overflowLimit = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT));
    if (recvBatchSize > recvQueueSize / 2) {
        throw new IllegalArgumentException(Config.TOPOLOGY_PRODUCER_BATCH_SIZE + ":" + recvBatchSize + " is greater than half of " + Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE + ":" + recvQueueSize);
    }
    IWaitStrategy backPressureWaitStrategy = IWaitStrategy.createBackPressureWaitStrategy(topologyConf);
    Map<List<Long>, JCQueue> receiveQueueMap = new HashMap<>();
    for (List<Long> executor : executors) {
        List<Integer> taskIds = StormCommon.executorIdToTasks(executor);
        int taskId = taskIds.get(0);
        String compId;
        if (taskId == Constants.SYSTEM_TASK_ID) {
            compId = Constants.SYSTEM_COMPONENT_ID;
        } else {
            compId = taskToComponent.get(taskId);
        }
        receiveQueueMap.put(executor, new JCQueue("receive-queue" + executor.toString(), "receive-queue", recvQueueSize, overflowLimit, recvBatchSize, backPressureWaitStrategy, this.getTopologyId(), compId, taskIds, this.getPort(), metricRegistry));
    }
    return receiveQueueMap;
}

Example 7 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class ExecutorTransferMultiThreadingTest method setup.

@Before
public void setup() throws NoSuchFieldException {
    topoConf = Utils.readStormConfig();
    String topologyId = "multi-threaded-topo-test";
    StormTopology stormTopology = createStormTopology();
    WorkerTopologyContext workerTopologyContext = mock(WorkerTopologyContext.class);
    when(workerTopologyContext.getRawTopology()).thenReturn(stormTopology);
    when(workerTopologyContext.getComponentId(selfTaskId)).thenReturn(sourceComp);
    when(workerTopologyContext.getComponentId(remoteTaskId)).thenReturn(destComp);
    workerState = mock(WorkerState.class);
    when(workerState.getWorkerTopologyContext()).thenReturn(workerTopologyContext);
    Map<Integer, JCQueue> receiveQMap = new HashMap<>();
    // local recvQ is not important in this test; simple mock it
    receiveQMap.put(selfTaskId, mock(JCQueue.class));
    when(workerState.getLocalReceiveQueues()).thenReturn(receiveQMap);
    when(workerState.getTopologyId()).thenReturn(topologyId);
    when(workerState.getPort()).thenReturn(6701);
    when(workerState.getMetricRegistry()).thenReturn(new StormMetricRegistry());
    when(workerState.tryTransferRemote(any(), any(), any())).thenCallRealMethod();
    // the actual worker transfer queue to be used in this test
    // taskId for worker transfer queue should be -1.
    // But there is already one worker transfer queue initialized by WorkerTransfer class (taskId=-1).
    // However the taskId is only used for metrics and it is not important here. Making it -100 to avoid collision.
    transferQueue = new JCQueue("worker-transfer-queue", "worker-transfer-queue", 1024, 0, 1, new WaitStrategyPark(100), workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections.singletonList(-100), workerState.getPort(), workerState.getMetricRegistry());
    // Replace the transferQueue inside WorkerTransfer (inside WorkerState) with the customized transferQueue to be used in this test
    WorkerTransfer workerTransfer = new WorkerTransfer(workerState, topoConf, 2);
    FieldSetter.setField(workerTransfer, workerTransfer.getClass().getDeclaredField("transferQueue"), transferQueue);
    FieldSetter.setField(workerState, workerState.getClass().getDeclaredField("workerTransfer"), workerTransfer);
    generalTopologyContext = mock(GeneralTopologyContext.class);
}

Example 8 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testSetBackpressureWithExistingBackpressure.

@Test
public void testSetBackpressureWithExistingBackpressure() {
    int taskId = 1;
    JCQueue queue = mock(JCQueue.class);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of(taskId, queue), new StormMetricRegistry(), ImmutableMap.of(taskId, "component-1"));
    BackpressureState state = tracker.getBackpressureState(taskId);
    tracker.recordBackPressure(state);
    boolean backpressureChanged = tracker.recordBackPressure(state);
    BackPressureStatus status = tracker.getCurrStatus();
    assertThat(backpressureChanged, is(false));
    assertThat(status.workerId, is(WORKER_ID));
    assertThat(status.bpTasks, contains(taskId));
}

Example 9 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testSetLastOverflowCount.

@Test
public void testSetLastOverflowCount() {
    int taskId = 1;
    int overflow = 5;
    JCQueue queue = mock(JCQueue.class);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of(taskId, queue), new StormMetricRegistry(), ImmutableMap.of(taskId, "component-1"));
    BackpressureState state = tracker.getBackpressureState(taskId);
    tracker.recordBackPressure(state);
    tracker.setLastOverflowCount(state, overflow);
    BackpressureState retrievedState = tracker.getBackpressureState(taskId);
    int lastOverflowCount = tracker.getLastOverflowCount(retrievedState);
    assertThat(lastOverflowCount, is(overflow));
}

Example 10 with JCQueue

use of org.apache.storm.utils.JCQueue in project storm by apache.

the class BackPressureTrackerTest method testGetBackpressure.

@Test
public void testGetBackpressure() {
    int taskIdNoBackPressure = 1;
    JCQueue noBackPressureQueue = mock(JCQueue.class);
    BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, Collections.singletonMap(taskIdNoBackPressure, noBackPressureQueue), new StormMetricRegistry(), Collections.singletonMap(taskIdNoBackPressure, "testComponent"));
    BackPressureStatus status = tracker.getCurrStatus();
    assertThat(status.workerId, is(WORKER_ID));
    assertThat(status.nonBpTasks, contains(taskIdNoBackPressure));
    assertThat(status.bpTasks, is(empty()));
}