Examples with Load org.apache.storm.grouping.Load used on opensource projects

Example 1 with Load

use of org.apache.storm.grouping.Load in project storm by apache.

the class WorkerState method refreshLoad.

public void refreshLoad() {
    Set<Integer> remoteTasks = Sets.difference(new HashSet<Integer>(outboundTasks), new HashSet<>(taskIds));
    Long now = System.currentTimeMillis();
    Map<Integer, Double> localLoad = shortExecutorReceiveQueueMap.entrySet().stream().collect(Collectors.toMap((Function<Map.Entry<Integer, DisruptorQueue>, Integer>) Map.Entry::getKey, (Function<Map.Entry<Integer, DisruptorQueue>, Double>) entry -> {
        DisruptorQueue.QueueMetrics qMetrics = entry.getValue().getMetrics();
        return ((double) qMetrics.population()) / qMetrics.capacity();
    }));
    Map<Integer, Load> remoteLoad = new HashMap<>();
    cachedNodeToPortSocket.get().values().stream().forEach(conn -> remoteLoad.putAll(conn.getLoad(remoteTasks)));
    loadMapping.setLocal(localLoad);
    loadMapping.setRemote(remoteLoad);
    if (now > nextUpdate.get()) {
        receiver.sendLoadMetrics(localLoad);
        nextUpdate.set(now + LOAD_REFRESH_INTERVAL_MS);
    }
}

Example 2 with Load

use of org.apache.storm.grouping.Load in project storm by apache.

the class NettyTest method doTestLoad.

private void doTestLoad(Map<String, Object> stormConf) throws Exception {
    LOG.info("2 test load");
    String reqMessage = "0123456789abcdefghijklmnopqrstuvwxyz";
    IContext context = TransportFactory.makeContext(stormConf, null);
    try {
        AtomicReference<TaskMessage> response = new AtomicReference<>();
        try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), null);
            IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) {
            waitUntilReady(client, server);
            byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8);
            send(client, taskId, messageBytes);
            /*
                 * This test sends a broadcast to all connected clients from the server, so we need to wait until the server has registered
                 * the client as connected before sending load metrics.
                 *
                 * It's not enough to wait until the client reports that the channel is open, because the server event loop may not have
                 * finished running channelActive for the new channel. If we send metrics too early, the server will broadcast to no one.
                 *
                 * By waiting for the response here, we ensure that the client will be registered at the server before we send load metrics.
                 */
            waitForNotNull(response);
            Map<Integer, Double> taskToLoad = new HashMap<>();
            taskToLoad.put(1, 0.0);
            taskToLoad.put(2, 1.0);
            server.sendLoadMetrics(taskToLoad);
            List<Integer> tasks = new ArrayList<>();
            tasks.add(1);
            tasks.add(2);
            Testing.whileTimeout(Testing.TEST_TIMEOUT_MS, () -> client.getLoad(tasks).isEmpty(), sleep());
            Map<Integer, Load> load = client.getLoad(tasks);
            assertThat(load.get(1).getBoltLoad(), is(0.0));
            assertThat(load.get(2).getBoltLoad(), is(1.0));
        }
    } finally {
        context.term();
    }
}

Example 3 with Load

use of org.apache.storm.grouping.Load in project storm by apache.

the class Client method getLoad.

@Override
public Map<Integer, Load> getLoad(Collection<Integer> tasks) {
    Map<Integer, Double> loadCache = serverLoad;
    Map<Integer, Load> ret = new HashMap<>();
    if (loadCache != null) {
        double clientLoad = Math.min(pendingMessages.get(), 1024) / 1024.0;
        for (Integer task : tasks) {
            Double found = loadCache.get(task);
            if (found != null) {
                ret.put(task, new Load(true, found, clientLoad));
            }
        }
    }
    return ret;
}

Example 4 with Load

use of org.apache.storm.grouping.Load in project storm by apache.

the class WorkerState method refreshLoad.

public void refreshLoad(List<IRunningExecutor> execs) {
    Set<Integer> remoteTasks = Sets.difference(new HashSet<>(outboundTasks), new HashSet<>(localTaskIds));
    Map<Integer, Double> localLoad = new HashMap<>();
    for (IRunningExecutor exec : execs) {
        double receiveLoad = exec.getReceiveQueue().getQueueLoad();
        localLoad.put(exec.getExecutorId().get(0).intValue(), receiveLoad);
    }
    Map<Integer, Load> remoteLoad = new HashMap<>();
    cachedNodeToPortSocket.get().values().stream().forEach(conn -> remoteLoad.putAll(conn.getLoad(remoteTasks)));
    loadMapping.setLocal(localLoad);
    loadMapping.setRemote(remoteLoad);
    Long now = System.currentTimeMillis();
    if (now > nextLoadUpdate.get()) {
        receiver.sendLoadMetrics(localLoad);
        nextLoadUpdate.set(now + LOAD_REFRESH_INTERVAL_MS);
    }
}