Examples with WorkerResources org.apache.storm.generated.WorkerResources used on opensource projects

Example 1 with WorkerResources

use of org.apache.storm.generated.WorkerResources in project storm by apache.

the class Nimbus method getWorkerResourcesForTopology.

private Map<WorkerSlot, WorkerResources> getWorkerResourcesForTopology(String topoId) {
    Map<WorkerSlot, WorkerResources> ret = idToWorkerResources.get().get(topoId);
    if (ret == null) {
        IStormClusterState state = stormClusterState;
        ret = new HashMap<>();
        Assignment assignment = state.assignmentInfo(topoId, null);
        if (assignment != null && assignment.is_set_worker_resources()) {
            for (Entry<NodeInfo, WorkerResources> entry : assignment.get_worker_resources().entrySet()) {
                NodeInfo ni = entry.getKey();
                WorkerSlot slot = new WorkerSlot(ni.get_node(), ni.get_port_iterator().next());
                ret.put(slot, entry.getValue());
            }
            idToWorkerResources.getAndUpdate(new Assoc<>(topoId, ret));
        }
    }
    return ret;
}

Example 2 with WorkerResources

use of org.apache.storm.generated.WorkerResources in project storm by apache.

the class Nimbus method mkAssignments.

private void mkAssignments(String scratchTopoId) throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping assignments");
        return;
    }
    // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {}
    // filter out ones which have a executor timeout
    // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors should be in each slot (e.g., 4, 4, 4, 5)
    // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots
    // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be reassigned to. worst comes to worse the executor will timeout and won't assign here next time around
    IStormClusterState state = stormClusterState;
    //read all the topologies
    Map<String, StormBase> bases;
    Map<String, TopologyDetails> tds = new HashMap<>();
    synchronized (submitLock) {
        bases = state.topologyBases();
        for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
            Entry<String, StormBase> entry = it.next();
            String id = entry.getKey();
            try {
                tds.put(id, readTopologyDetails(id, entry.getValue()));
            } catch (KeyNotFoundException e) {
                //A race happened and it is probably not running
                it.remove();
            }
        }
    }
    Topologies topologies = new Topologies(tds);
    List<String> assignedTopologyIds = state.assignments(null);
    Map<String, Assignment> existingAssignments = new HashMap<>();
    for (String id : assignedTopologyIds) {
        // will be treated as free slot in the scheduler code.
        if (!id.equals(scratchTopoId)) {
            existingAssignments.put(id, state.assignmentInfo(id, null));
        }
    }
    // make the new assignments for topologies
    Map<String, SchedulerAssignment> newSchedulerAssignments = null;
    synchronized (schedLock) {
        newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
        Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeNewTopoToExecToNodePort(newSchedulerAssignments, existingAssignments);
        for (String id : assignedTopologyIds) {
            if (!topologyToExecutorToNodePort.containsKey(id)) {
                topologyToExecutorToNodePort.put(id, null);
            }
        }
        Map<String, Map<List<Object>, List<Double>>> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments);
        int nowSecs = Time.currentTimeSecs();
        Map<String, SupervisorDetails> basicSupervisorDetailsMap = basicSupervisorDetailsMap(state);
        //construct the final Assignments by adding start-times etc into it
        Map<String, Assignment> newAssignments = new HashMap<>();
        for (Entry<String, Map<List<Long>, List<Object>>> entry : topologyToExecutorToNodePort.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            Set<String> allNodes = new HashSet<>();
            if (execToNodePort != null) {
                for (List<Object> nodePort : execToNodePort.values()) {
                    allNodes.add((String) nodePort.get(0));
                }
            }
            Map<String, String> allNodeHost = new HashMap<>();
            if (existingAssignment != null) {
                allNodeHost.putAll(existingAssignment.get_node_host());
            }
            for (String node : allNodes) {
                String host = inimbus.getHostName(basicSupervisorDetailsMap, node);
                if (host != null) {
                    allNodeHost.put(node, host);
                }
            }
            Map<List<Long>, NodeInfo> execNodeInfo = null;
            if (existingAssignment != null) {
                execNodeInfo = existingAssignment.get_executor_node_port();
            }
            List<List<Long>> reassignExecutors = changedExecutors(execNodeInfo, execToNodePort);
            Map<List<Long>, Long> startTimes = new HashMap<>();
            if (existingAssignment != null) {
                startTimes.putAll(existingAssignment.get_executor_start_time_secs());
            }
            for (List<Long> id : reassignExecutors) {
                startTimes.put(id, (long) nowSecs);
            }
            Map<List<Object>, List<Double>> workerToResources = newAssignedWorkerToResources.get(topoId);
            Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR));
            Map<String, String> justAssignedKeys = new HashMap<>(allNodeHost);
            //Modifies justAssignedKeys
            justAssignedKeys.keySet().retainAll(allNodes);
            newAssignment.set_node_host(justAssignedKeys);
            //convert NodePort to NodeInfo (again!!!).
            Map<List<Long>, NodeInfo> execToNodeInfo = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                List<Object> nodePort = execAndNodePort.getValue();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                execToNodeInfo.put(execAndNodePort.getKey(), ni);
            }
            newAssignment.set_executor_node_port(execToNodeInfo);
            newAssignment.set_executor_start_time_secs(startTimes);
            //do another conversion (lets just make this all common)
            Map<NodeInfo, WorkerResources> workerResources = new HashMap<>();
            for (Entry<List<Object>, List<Double>> wr : workerToResources.entrySet()) {
                List<Object> nodePort = wr.getKey();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                List<Double> r = wr.getValue();
                WorkerResources resources = new WorkerResources();
                resources.set_mem_on_heap(r.get(0));
                resources.set_mem_off_heap(r.get(1));
                resources.set_cpu(r.get(2));
                workerResources.put(ni, resources);
            }
            newAssignment.set_worker_resources(workerResources);
            newAssignments.put(topoId, newAssignment);
        }
        if (!newAssignments.equals(existingAssignments)) {
            LOG.debug("RESETTING id->resources and id->worker-resources cache!");
            idToResources.set(new HashMap<>());
            idToWorkerResources.set(new HashMap<>());
        }
        // only log/set when there's been a change to the assignment
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            //NOT Used TopologyDetails topologyDetails = topologies.getById(topoId);
            if (assignment.equals(existingAssignment)) {
                LOG.debug("Assignment for {} hasn't changed", topoId);
            } else {
                LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment);
                state.setAssignment(topoId, assignment);
            }
        }
        Map<String, Collection<WorkerSlot>> addedSlots = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment == null) {
                existingAssignment = new Assignment();
                existingAssignment.set_executor_node_port(new HashMap<>());
                existingAssignment.set_executor_start_time_secs(new HashMap<>());
            }
            Set<WorkerSlot> newSlots = newlyAddedSlots(existingAssignment, assignment);
            addedSlots.put(topoId, newSlots);
        }
        inimbus.assignSlots(topologies, addedSlots);
    }
}

Example 3 with WorkerResources

use of org.apache.storm.generated.WorkerResources in project storm by apache.

the class Nimbus method computeTopologyToSchedulerAssignment.

/**
     * Convert assignment information in zk to SchedulerAssignment, so it can be used by scheduler api.
     * @param existingAssignments current assignments
     * @param topologyToAliveExecutors executors that are alive
     * @return topo ID to schedulerAssignment
     */
private Map<String, SchedulerAssignmentImpl> computeTopologyToSchedulerAssignment(Map<String, Assignment> existingAssignments, Map<String, Set<List<Integer>>> topologyToAliveExecutors) {
    Map<String, SchedulerAssignmentImpl> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> aliveExecutors = topologyToAliveExecutors.get(topoId);
        Map<List<Long>, NodeInfo> execToNodePort = assignment.get_executor_node_port();
        Map<NodeInfo, WorkerResources> workerToResources = assignment.get_worker_resources();
        Map<NodeInfo, WorkerSlot> nodePortToSlot = new HashMap<>();
        for (Entry<NodeInfo, WorkerResources> nodeAndResources : workerToResources.entrySet()) {
            NodeInfo info = nodeAndResources.getKey();
            WorkerResources resources = nodeAndResources.getValue();
            WorkerSlot slot = new WorkerSlot(info.get_node(), info.get_port_iterator().next(), resources.get_mem_on_heap(), resources.get_mem_off_heap(), resources.get_cpu());
            nodePortToSlot.put(info, slot);
        }
        Map<ExecutorDetails, WorkerSlot> execToSlot = new HashMap<>();
        for (Entry<List<Long>, NodeInfo> execAndNodePort : execToNodePort.entrySet()) {
            List<Integer> exec = asIntExec(execAndNodePort.getKey());
            NodeInfo info = execAndNodePort.getValue();
            if (aliveExecutors.contains(exec)) {
                execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot.get(info));
            }
        }
        ret.put(topoId, new SchedulerAssignmentImpl(topoId, execToSlot));
    }
    return ret;
}

Example 4 with WorkerResources

use of org.apache.storm.generated.WorkerResources in project storm by apache.

the class Nimbus method getSupervisorPageInfo.

@Override
public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getSupervisorPageInfoCalls.mark();
        IStormClusterState state = stormClusterState;
        Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
        Map<String, List<String>> hostToSuperId = new HashMap<>();
        for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
            String h = entry.getValue().get_hostname();
            List<String> superIds = hostToSuperId.get(h);
            if (superIds == null) {
                superIds = new ArrayList<>();
                hostToSuperId.put(h, superIds);
            }
            superIds.add(entry.getKey());
        }
        List<String> supervisorIds = null;
        if (superId == null) {
            supervisorIds = hostToSuperId.get(host);
        } else {
            supervisorIds = Arrays.asList(superId);
        }
        SupervisorPageInfo pageInfo = new SupervisorPageInfo();
        Map<String, Assignment> topoToAssignment = state.topologyAssignments();
        for (String sid : supervisorIds) {
            SupervisorInfo info = superInfos.get(sid);
            LOG.info("SIDL {} SI: {} ALL: {}", sid, info, superInfos);
            SupervisorSummary supSum = makeSupervisorSummary(sid, info);
            pageInfo.add_to_supervisor_summaries(supSum);
            List<String> superTopologies = topologiesOnSupervisor(topoToAssignment, sid);
            Set<String> userTopologies = filterAuthorized("getTopology", superTopologies);
            for (String topoId : superTopologies) {
                CommonTopoInfo common = getCommonTopoInfo(topoId, "getSupervisorPageInfo");
                String topoName = common.topoName;
                Assignment assignment = common.assignment;
                Map<List<Integer>, Map<String, Object>> beats = common.beats;
                Map<Integer, String> taskToComp = common.taskToComponent;
                Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
                Map<String, String> nodeToHost;
                if (assignment != null) {
                    Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
                    for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                        NodeInfo ni = entry.getValue();
                        List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                        exec2NodePort.put(entry.getKey(), nodePort);
                    }
                    nodeToHost = assignment.get_node_host();
                } else {
                    nodeToHost = Collections.emptyMap();
                }
                Map<WorkerSlot, WorkerResources> workerResources = getWorkerResourcesForTopology(topoId);
                boolean isAllowed = userTopologies.contains(topoId);
                for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid)) {
                    pageInfo.add_to_worker_summaries(workerSummary);
                }
            }
        }
        return pageInfo;
    } catch (Exception e) {
        LOG.warn("Get super page info exception. (super id='{}')", superId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 5 with WorkerResources

use of org.apache.storm.generated.WorkerResources in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        int launchTimeSecs = common.launchTimeSecs;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        Map<String, Object> topoConf = common.topoConf;
        StormBase base = common.base;
        if (base == null) {
            throw new NotAliveException(topoId);
        }
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, //this is the topology page, so we know the user is authorized
            true);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        Map<String, Map<String, Double>> spoutResources = ResourceUtils.getSpoutsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_spout_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf));
        }
        Map<String, Map<String, Double>> boltResources = ResourceUtils.getBoltsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_bolt_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf));
        }
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}