Search in sources :

Example 21 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method computeTopologyToSchedulerAssignment.

/**
     * Convert assignment information in zk to SchedulerAssignment, so it can be used by scheduler api.
     * @param existingAssignments current assignments
     * @param topologyToAliveExecutors executors that are alive
     * @return topo ID to schedulerAssignment
     */
private Map<String, SchedulerAssignmentImpl> computeTopologyToSchedulerAssignment(Map<String, Assignment> existingAssignments, Map<String, Set<List<Integer>>> topologyToAliveExecutors) {
    Map<String, SchedulerAssignmentImpl> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> aliveExecutors = topologyToAliveExecutors.get(topoId);
        Map<List<Long>, NodeInfo> execToNodePort = assignment.get_executor_node_port();
        Map<NodeInfo, WorkerResources> workerToResources = assignment.get_worker_resources();
        Map<NodeInfo, WorkerSlot> nodePortToSlot = new HashMap<>();
        for (Entry<NodeInfo, WorkerResources> nodeAndResources : workerToResources.entrySet()) {
            NodeInfo info = nodeAndResources.getKey();
            WorkerResources resources = nodeAndResources.getValue();
            WorkerSlot slot = new WorkerSlot(info.get_node(), info.get_port_iterator().next(), resources.get_mem_on_heap(), resources.get_mem_off_heap(), resources.get_cpu());
            nodePortToSlot.put(info, slot);
        }
        Map<ExecutorDetails, WorkerSlot> execToSlot = new HashMap<>();
        for (Entry<List<Long>, NodeInfo> execAndNodePort : execToNodePort.entrySet()) {
            List<Integer> exec = asIntExec(execAndNodePort.getKey());
            NodeInfo info = execAndNodePort.getValue();
            if (aliveExecutors.contains(exec)) {
                execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot.get(info));
            }
        }
        ret.put(topoId, new SchedulerAssignmentImpl(topoId, execToSlot));
    }
    return ret;
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) WorkerResources(org.apache.storm.generated.WorkerResources) Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) SchedulerAssignmentImpl(org.apache.storm.scheduler.SchedulerAssignmentImpl) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) NodeInfo(org.apache.storm.generated.NodeInfo) ArrayList(java.util.ArrayList) List(java.util.List)

Example 22 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method getSupervisorPageInfo.

@Override
public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getSupervisorPageInfoCalls.mark();
        IStormClusterState state = stormClusterState;
        Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
        Map<String, List<String>> hostToSuperId = new HashMap<>();
        for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
            String h = entry.getValue().get_hostname();
            List<String> superIds = hostToSuperId.get(h);
            if (superIds == null) {
                superIds = new ArrayList<>();
                hostToSuperId.put(h, superIds);
            }
            superIds.add(entry.getKey());
        }
        List<String> supervisorIds = null;
        if (superId == null) {
            supervisorIds = hostToSuperId.get(host);
        } else {
            supervisorIds = Arrays.asList(superId);
        }
        SupervisorPageInfo pageInfo = new SupervisorPageInfo();
        Map<String, Assignment> topoToAssignment = state.topologyAssignments();
        for (String sid : supervisorIds) {
            SupervisorInfo info = superInfos.get(sid);
            LOG.info("SIDL {} SI: {} ALL: {}", sid, info, superInfos);
            SupervisorSummary supSum = makeSupervisorSummary(sid, info);
            pageInfo.add_to_supervisor_summaries(supSum);
            List<String> superTopologies = topologiesOnSupervisor(topoToAssignment, sid);
            Set<String> userTopologies = filterAuthorized("getTopology", superTopologies);
            for (String topoId : superTopologies) {
                CommonTopoInfo common = getCommonTopoInfo(topoId, "getSupervisorPageInfo");
                String topoName = common.topoName;
                Assignment assignment = common.assignment;
                Map<List<Integer>, Map<String, Object>> beats = common.beats;
                Map<Integer, String> taskToComp = common.taskToComponent;
                Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
                Map<String, String> nodeToHost;
                if (assignment != null) {
                    Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
                    for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                        NodeInfo ni = entry.getValue();
                        List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                        exec2NodePort.put(entry.getKey(), nodePort);
                    }
                    nodeToHost = assignment.get_node_host();
                } else {
                    nodeToHost = Collections.emptyMap();
                }
                Map<WorkerSlot, WorkerResources> workerResources = getWorkerResourcesForTopology(topoId);
                boolean isAllowed = userTopologies.contains(topoId);
                for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid)) {
                    pageInfo.add_to_worker_summaries(workerSummary);
                }
            }
        }
        return pageInfo;
    } catch (Exception e) {
        LOG.warn("Get super page info exception. (super id='{}')", superId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) SupervisorSummary(org.apache.storm.generated.SupervisorSummary) SupervisorInfo(org.apache.storm.generated.SupervisorInfo) Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) WorkerResources(org.apache.storm.generated.WorkerResources) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) WorkerSummary(org.apache.storm.generated.WorkerSummary) SupervisorPageInfo(org.apache.storm.generated.SupervisorPageInfo) NodeInfo(org.apache.storm.generated.NodeInfo) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 23 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        int launchTimeSecs = common.launchTimeSecs;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        Map<String, Object> topoConf = common.topoConf;
        StormBase base = common.base;
        if (base == null) {
            throw new NotAliveException(topoId);
        }
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, //this is the topology page, so we know the user is authorized
            true);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        Map<String, Map<String, Double>> spoutResources = ResourceUtils.getSpoutsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_spout_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf));
        }
        Map<String, Map<String, Double>> boltResources = ResourceUtils.getBoltsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_bolt_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf));
        }
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) StormTopology(org.apache.storm.generated.StormTopology) StormBase(org.apache.storm.generated.StormBase) DebugOptions(org.apache.storm.generated.DebugOptions) Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) NotAliveException(org.apache.storm.generated.NotAliveException) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) CommonAggregateStats(org.apache.storm.generated.CommonAggregateStats) WorkerResources(org.apache.storm.generated.WorkerResources) ComponentAggregateStats(org.apache.storm.generated.ComponentAggregateStats) TopologyPageInfo(org.apache.storm.generated.TopologyPageInfo) DataPoint(org.apache.storm.metric.api.DataPoint) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) WorkerSummary(org.apache.storm.generated.WorkerSummary) NodeInfo(org.apache.storm.generated.NodeInfo) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 24 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method computeTopoToNodePortToResources.

/**
     * convert {topology-id -> SchedulerAssignment} to
     *         {topology-id -> {[node port] [mem-on-heap mem-off-heap cpu]}}
     * Make sure this can deal with other non-RAS schedulers
     * later we may further support map-for-any-resources
     * @param schedAssignments the assignments
     * @return  {topology-id {[node port] [mem-on-heap mem-off-heap cpu]}}
     */
private static Map<String, Map<List<Object>, List<Double>>> computeTopoToNodePortToResources(Map<String, SchedulerAssignment> schedAssignments) {
    Map<String, Map<List<Object>, List<Double>>> ret = new HashMap<>();
    for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
        Map<List<Object>, List<Double>> nodePortToResources = new HashMap<>();
        for (WorkerSlot slot : schedEntry.getValue().getExecutorToSlot().values()) {
            List<Object> nodePort = new ArrayList<>(2);
            nodePort.add(slot.getNodeId());
            nodePort.add((long) slot.getPort());
            List<Double> resources = new ArrayList<>(3);
            resources.add(slot.getAllocatedMemOnHeap());
            resources.add(slot.getAllocatedMemOffHeap());
            resources.add(slot.getAllocatedCpu());
            nodePortToResources.put(nodePort, resources);
        }
        ret.put(schedEntry.getKey(), nodePortToResources);
    }
    return ret;
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 25 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method computeTopoToExecToNodePort.

/**
     * convert {topology-id -> SchedulerAssignment} to
     *         {topology-id -> {executor [node port]}}
     * @return
     */
private static Map<String, Map<List<Long>, List<Object>>> computeTopoToExecToNodePort(Map<String, SchedulerAssignment> schedAssignments) {
    Map<String, Map<List<Long>, List<Object>>> ret = new HashMap<>();
    for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
        Map<List<Long>, List<Object>> execToNodePort = new HashMap<>();
        for (Entry<ExecutorDetails, WorkerSlot> execAndNodePort : schedEntry.getValue().getExecutorToSlot().entrySet()) {
            ExecutorDetails exec = execAndNodePort.getKey();
            WorkerSlot slot = execAndNodePort.getValue();
            List<Long> listExec = new ArrayList<>(2);
            listExec.add((long) exec.getStartTask());
            listExec.add((long) exec.getEndTask());
            List<Object> nodePort = new ArrayList<>(2);
            nodePort.add(slot.getNodeId());
            nodePort.add((long) slot.getPort());
            execToNodePort.put(listExec, nodePort);
        }
        ret.put(schedEntry.getKey(), execToNodePort);
    }
    return ret;
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) AtomicLong(java.util.concurrent.atomic.AtomicLong) ArrayList(java.util.ArrayList) List(java.util.List) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Aggregations

WorkerSlot (org.apache.storm.scheduler.WorkerSlot)45 HashMap (java.util.HashMap)33 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)23 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)23 ArrayList (java.util.ArrayList)22 Map (java.util.Map)22 List (java.util.List)17 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)16 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)15 HashSet (java.util.HashSet)13 SchedulerAssignmentImpl (org.apache.storm.scheduler.SchedulerAssignmentImpl)11 Topologies (org.apache.storm.scheduler.Topologies)11 Cluster (org.apache.storm.scheduler.Cluster)10 LinkedList (java.util.LinkedList)9 Config (org.apache.storm.Config)9 INimbus (org.apache.storm.scheduler.INimbus)9 Test (org.junit.Test)9 Collection (java.util.Collection)8 WorkerResources (org.apache.storm.generated.WorkerResources)8 ImmutableMap (com.google.common.collect.ImmutableMap)6