Examples with NodeInfo org.apache.storm.generated.NodeInfo used on opensource projects

Example 1 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class Nimbus method getTopologyInfoWithOpts.

@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyInfoWithOptsCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
        if (common.base == null) {
            throw new NotAliveException(topoId);
        }
        IStormClusterState state = stormClusterState;
        NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
        Map<String, List<ErrorInfo>> errors = new HashMap<>();
        for (String component : common.allComponents) {
            switch(numErrChoice) {
                case NONE:
                    errors.put(component, Collections.emptyList());
                    break;
                case ONE:
                    List<ErrorInfo> errList = new ArrayList<>();
                    ErrorInfo info = state.lastError(topoId, component);
                    if (info != null) {
                        errList.add(info);
                    }
                    errors.put(component, errList);
                    break;
                case ALL:
                    errors.put(component, state.errors(topoId, component));
                    break;
                default:
                    LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                    errors.put(component, state.errors(topoId, component));
                    break;
            }
        }
        List<ExecutorSummary> summaries = new ArrayList<>();
        if (common.assignment != null) {
            for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                ExecutorInfo execInfo = toExecInfo(entry.getKey());
                String host = entry.getValue().get_node();
                Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
                if (heartbeat == null) {
                    heartbeat = Collections.emptyMap();
                }
                ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
                //heartbeats "stats"
                Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
                if (hb != null) {
                    Map ex = (Map) hb.get("stats");
                    if (ex != null) {
                        ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                        summ.set_stats(stats);
                    }
                }
                summaries.add(summ);
            }
        }
        TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
        if (common.base.is_set_owner()) {
            topoInfo.set_owner(common.base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, common.base);
        if (resources != null) {
            topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoInfo.set_requested_cpu(resources.getRequestedCpu());
            topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        if (common.base.is_set_component_debug()) {
            topoInfo.set_component_debug(common.base.get_component_debug());
        }
        topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        return topoInfo;
    } catch (Exception e) {
        LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 2 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class WorkerState method refreshConnections.

public void refreshConnections() {
    Assignment assignment = null;
    try {
        assignment = getLocalAssignment(stormClusterState, topologyId);
    } catch (Exception e) {
        LOG.warn("Failed to read assignment. This should only happen when topology is shutting down.", e);
    }
    suicideIfLocalAssignmentsChanged(assignment);
    Set<NodeInfo> neededConnections = new HashSet<>();
    Map<Integer, NodeInfo> newTaskToNodePort = new HashMap<>();
    if (null != assignment) {
        Map<Integer, NodeInfo> taskToNodePort = StormCommon.taskToNodeport(assignment.get_executor_node_port());
        for (Map.Entry<Integer, NodeInfo> taskToNodePortEntry : taskToNodePort.entrySet()) {
            Integer task = taskToNodePortEntry.getKey();
            if (outboundTasks.contains(task)) {
                newTaskToNodePort.put(task, taskToNodePortEntry.getValue());
                if (!localTaskIds.contains(task)) {
                    neededConnections.add(taskToNodePortEntry.getValue());
                }
            }
        }
    }
    final Set<NodeInfo> currentConnections = cachedNodeToPortSocket.get().keySet();
    final Set<NodeInfo> newConnections = Sets.difference(neededConnections, currentConnections);
    final Set<NodeInfo> removeConnections = Sets.difference(currentConnections, neededConnections);
    Map<String, String> nodeHost = assignment != null ? assignment.get_node_host() : null;
    // Add new connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        for (NodeInfo nodeInfo : newConnections) {
            next.put(nodeInfo, mqContext.connect(topologyId, // Host
            nodeHost.get(nodeInfo.get_node()), // Port
            nodeInfo.get_port().iterator().next().intValue(), workerTransfer.getRemoteBackPressureStatus()));
        }
        return next;
    });
    try {
        endpointSocketLock.writeLock().lock();
        cachedTaskToNodePort.set(newTaskToNodePort);
    } finally {
        endpointSocketLock.writeLock().unlock();
    }
    // It is okay that cachedNodeToHost can be temporarily out of sync with cachedTaskToNodePort
    if (nodeHost != null) {
        cachedNodeToHost.set(nodeHost);
    } else {
        cachedNodeToHost.set(new HashMap<>());
    }
    for (NodeInfo nodeInfo : removeConnections) {
        cachedNodeToPortSocket.get().get(nodeInfo).close();
    }
    // Remove old connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        removeConnections.forEach(next::remove);
        return next;
    });
}

Example 3 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class WorkerState method readWorkerExecutors.

private List<List<Long>> readWorkerExecutors(String assignmentId, int port, Assignment assignment) {
    List<List<Long>> executorsAssignedToThisWorker = new ArrayList<>();
    executorsAssignedToThisWorker.add(Constants.SYSTEM_EXECUTOR_ID);
    Map<List<Long>, NodeInfo> executorToNodePort = assignment.get_executor_node_port();
    for (Map.Entry<List<Long>, NodeInfo> entry : executorToNodePort.entrySet()) {
        NodeInfo nodeInfo = entry.getValue();
        if (nodeInfo.get_node().equals(assignmentId) && nodeInfo.get_port().iterator().next() == port) {
            executorsAssignedToThisWorker.add(entry.getKey());
        }
    }
    return executorsAssignedToThisWorker;
}

Example 4 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class Nimbus method changedExecutors.

private static List<List<Long>> changedExecutors(Map<List<Long>, NodeInfo> map, Map<List<Long>, List<Object>> newExecToNodePort) {
    HashMap<NodeInfo, List<List<Long>>> tmpSlotAssigned = map == null ? new HashMap<>() : Utils.reverseMap(map);
    HashMap<List<Object>, List<List<Long>>> slotAssigned = new HashMap<>();
    for (Entry<NodeInfo, List<List<Long>>> entry : tmpSlotAssigned.entrySet()) {
        NodeInfo ni = entry.getKey();
        List<Object> key = new ArrayList<>(2);
        key.add(ni.get_node());
        key.add(ni.get_port_iterator().next());
        List<List<Long>> value = new ArrayList<>(entry.getValue());
        value.sort(Comparator.comparing(a -> a.get(0)));
        slotAssigned.put(key, value);
    }
    HashMap<List<Object>, List<List<Long>>> tmpNewSlotAssigned = newExecToNodePort == null ? new HashMap<>() : Utils.reverseMap(newExecToNodePort);
    HashMap<List<Object>, List<List<Long>>> newSlotAssigned = new HashMap<>();
    for (Entry<List<Object>, List<List<Long>>> entry : tmpNewSlotAssigned.entrySet()) {
        List<List<Long>> value = new ArrayList<>(entry.getValue());
        value.sort(Comparator.comparing(a -> a.get(0)));
        newSlotAssigned.put(entry.getKey(), value);
    }
    Map<List<Object>, List<List<Long>>> diff = mapDiff(slotAssigned, newSlotAssigned);
    List<List<Long>> ret = new ArrayList<>();
    for (List<List<Long>> val : diff.values()) {
        ret.addAll(val);
    }
    return ret;
}

Example 5 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        StormBase base = common.base;
        if (base == null) {
            throw new WrappedNotAliveException(topoId);
        }
        String owner = base.get_owner();
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, // this is the topology page, so we know the user is authorized
            true, null, owner);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        if (topology.is_set_storm_version()) {
            topoPageInfo.set_storm_version(topology.get_storm_version());
        }
        Map<String, Object> topoConf = Utils.merge(conf, common.topoConf);
        addSpoutAggStats(topoPageInfo, topology, topoConf);
        addBoltAggStats(topoPageInfo, topology, topoConf, includeSys);
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        if (base.is_set_topology_version()) {
            topoPageInfo.set_topology_version(base.get_topology_version());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null && underlyingScheduler instanceof ResourceAwareScheduler) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
            topoPageInfo.set_requested_shared_off_heap_memory(resources.getRequestedSharedMemOffHeap());
            topoPageInfo.set_requested_regular_off_heap_memory(resources.getRequestedNonSharedMemOffHeap());
            topoPageInfo.set_requested_shared_on_heap_memory(resources.getRequestedSharedMemOnHeap());
            topoPageInfo.set_requested_regular_on_heap_memory(resources.getRequestedNonSharedMemOnHeap());
            topoPageInfo.set_assigned_shared_off_heap_memory(resources.getAssignedSharedMemOffHeap());
            topoPageInfo.set_assigned_regular_off_heap_memory(resources.getAssignedNonSharedMemOffHeap());
            topoPageInfo.set_assigned_shared_on_heap_memory(resources.getAssignedSharedMemOnHeap());
            topoPageInfo.set_assigned_regular_on_heap_memory(resources.getAssignedNonSharedMemOnHeap());
            topoPageInfo.set_assigned_generic_resources(resources.getAssignedGenericResources());
            topoPageInfo.set_requested_generic_resources(resources.getRequestedGenericResources());
        }
        int launchTimeSecs = common.launchTimeSecs;
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}