Examples with Assignment org.apache.storm.generated.Assignment used on opensource projects

Example 21 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method getOwnerResourceSummaries.

@Override
public List<OwnerResourceSummary> getOwnerResourceSummaries(String owner) throws AuthorizationException, TException {
    try {
        getOwnerResourceSummariesCalls.mark();
        checkAuthorization(null, null, "getOwnerResourceSummaries");
        IStormClusterState state = stormClusterState;
        Map<String, Assignment> topoIdToAssignments = state.assignmentsInfo();
        Map<String, StormBase> topoIdToBases = state.topologyBases();
        Map<String, Number> clusterSchedulerConfig = scheduler.config();
        // put [owner-> StormBase-list] mapping to ownerToBasesMap
        // if this owner (the input parameter) is null, add all the owners with stormbase and guarantees
        // else, add only this owner (the input paramter) to the map
        Map<String, List<StormBase>> ownerToBasesMap = new HashMap<>();
        if (owner == null) {
            // add all the owners to the map
            for (StormBase base : topoIdToBases.values()) {
                String baseOwner = base.get_owner();
                if (!ownerToBasesMap.containsKey(baseOwner)) {
                    List<StormBase> stormbases = new ArrayList<>();
                    stormbases.add(base);
                    ownerToBasesMap.put(baseOwner, stormbases);
                } else {
                    ownerToBasesMap.get(baseOwner).add(base);
                }
            }
            // in addition, add all the owners with guarantees
            List<String> ownersWithGuarantees = new ArrayList<>(clusterSchedulerConfig.keySet());
            for (String ownerWithGuarantees : ownersWithGuarantees) {
                if (!ownerToBasesMap.containsKey(ownerWithGuarantees)) {
                    ownerToBasesMap.put(ownerWithGuarantees, new ArrayList<>());
                }
            }
        } else {
            // only put this owner to the map
            List<StormBase> stormbases = new ArrayList<>();
            for (StormBase base : topoIdToBases.values()) {
                if (owner.equals(base.get_owner())) {
                    stormbases.add(base);
                }
            }
            ownerToBasesMap.put(owner, stormbases);
        }
        List<OwnerResourceSummary> ret = new ArrayList<>();
        // for each owner, get resources, configs, and aggregate
        for (Entry<String, List<StormBase>> ownerToBasesEntry : ownerToBasesMap.entrySet()) {
            String theOwner = ownerToBasesEntry.getKey();
            TopologyResources totalResourcesAggregate = new TopologyResources();
            int totalExecutors = 0;
            int totalWorkers = 0;
            int totalTasks = 0;
            for (StormBase base : ownerToBasesEntry.getValue()) {
                try {
                    String topoId = toTopoId(base.get_name());
                    TopologyResources resources = getResourcesForTopology(topoId, base);
                    totalResourcesAggregate = totalResourcesAggregate.add(resources);
                    Assignment ownerAssignment = topoIdToAssignments.get(topoId);
                    if (ownerAssignment != null && ownerAssignment.get_executor_node_port() != null) {
                        totalExecutors += ownerAssignment.get_executor_node_port().keySet().size();
                        totalWorkers += new HashSet(ownerAssignment.get_executor_node_port().values()).size();
                        for (List<Long> executorId : ownerAssignment.get_executor_node_port().keySet()) {
                            totalTasks += StormCommon.executorIdToTasks(executorId).size();
                        }
                    }
                } catch (NotAliveException e) {
                    LOG.warn("{} is not alive.", base.get_name());
                }
            }
            double requestedTotalMemory = totalResourcesAggregate.getRequestedMemOnHeap() + totalResourcesAggregate.getRequestedMemOffHeap();
            double assignedTotalMemory = totalResourcesAggregate.getAssignedMemOnHeap() + totalResourcesAggregate.getAssignedMemOffHeap();
            OwnerResourceSummary ownerResourceSummary = new OwnerResourceSummary(theOwner);
            ownerResourceSummary.set_total_topologies(ownerToBasesEntry.getValue().size());
            ownerResourceSummary.set_total_executors(totalExecutors);
            ownerResourceSummary.set_total_workers(totalWorkers);
            ownerResourceSummary.set_total_tasks(totalTasks);
            ownerResourceSummary.set_memory_usage(assignedTotalMemory);
            ownerResourceSummary.set_cpu_usage(totalResourcesAggregate.getAssignedCpu());
            ownerResourceSummary.set_requested_on_heap_memory(totalResourcesAggregate.getRequestedMemOnHeap());
            ownerResourceSummary.set_requested_off_heap_memory(totalResourcesAggregate.getRequestedMemOffHeap());
            ownerResourceSummary.set_requested_total_memory(requestedTotalMemory);
            ownerResourceSummary.set_requested_cpu(totalResourcesAggregate.getRequestedCpu());
            ownerResourceSummary.set_assigned_on_heap_memory(totalResourcesAggregate.getAssignedMemOnHeap());
            ownerResourceSummary.set_assigned_off_heap_memory(totalResourcesAggregate.getAssignedMemOffHeap());
            if (clusterSchedulerConfig.containsKey(theOwner)) {
                if (underlyingScheduler instanceof ResourceAwareScheduler) {
                    Map<String, Object> schedulerConfig = (Map) clusterSchedulerConfig.get(theOwner);
                    if (schedulerConfig != null) {
                        ownerResourceSummary.set_memory_guarantee((double) schedulerConfig.getOrDefault("memory", 0));
                        ownerResourceSummary.set_cpu_guarantee((double) schedulerConfig.getOrDefault("cpu", 0));
                        ownerResourceSummary.set_memory_guarantee_remaining(ownerResourceSummary.get_memory_guarantee() - ownerResourceSummary.get_memory_usage());
                        ownerResourceSummary.set_cpu_guarantee_remaining(ownerResourceSummary.get_cpu_guarantee() - ownerResourceSummary.get_cpu_usage());
                    }
                } else if (underlyingScheduler instanceof MultitenantScheduler) {
                    ownerResourceSummary.set_isolated_node_guarantee((int) clusterSchedulerConfig.getOrDefault(theOwner, 0));
                }
            }
            LOG.debug("{}", ownerResourceSummary.toString());
            ret.add(ownerResourceSummary);
        }
        return ret;
    } catch (Exception e) {
        LOG.warn("Get owner resource summaries exception. (owner = '{}')", owner);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 22 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method getTopologySummaryImpl.

private TopologySummary getTopologySummaryImpl(String topoId, StormBase base) throws IOException, TException {
    IStormClusterState state = stormClusterState;
    Assignment assignment = state.assignmentInfo(topoId, null);
    int numTasks = 0;
    int numExecutors = 0;
    int numWorkers = 0;
    if (assignment != null && assignment.is_set_executor_node_port()) {
        for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
            numTasks += StormCommon.executorIdToTasks(ids).size();
        }
        numExecutors = assignment.get_executor_node_port_size();
        numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
    }
    TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
    try {
        StormTopology topo = tryReadTopology(topoId, topoCache);
        if (topo != null && topo.is_set_storm_version()) {
            summary.set_storm_version(topo.get_storm_version());
        }
    } catch (NotAliveException e) {
    // Ignored it is not set
    }
    if (base.is_set_owner()) {
        summary.set_owner(base.get_owner());
    }
    if (base.is_set_topology_version()) {
        summary.set_topology_version(base.get_topology_version());
    }
    String status = idToSchedStatus.get().get(topoId);
    if (status != null) {
        summary.set_sched_status(status);
    }
    TopologyResources resources = getResourcesForTopology(topoId, base);
    if (resources != null) {
        summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
        summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
        summary.set_requested_cpu(resources.getRequestedCpu());
        summary.set_requested_generic_resources(resources.getRequestedGenericResources());
        summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
        summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
        summary.set_assigned_cpu(resources.getAssignedCpu());
        summary.set_assigned_generic_resources(resources.getAssignedGenericResources());
    }
    try {
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
    } catch (Exception e) {
        // This could fail if a blob gets deleted by mistake.  Don't crash nimbus.
        LOG.error("Unable to find blob entry", e);
    }
    return summary;
}

Example 23 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method computeTopologyToAliveExecutors.

/**
 * compute a topology-id -> alive executors map.
 *
 * @param existingAssignment  the current assignments
 * @param topologyToExecutors the executors for the current topologies
 * @param scratchTopologyId   the topology being rebalanced and should be excluded
 * @return the map of topology id to alive executors
 */
private Map<String, Set<List<Integer>>> computeTopologyToAliveExecutors(Map<String, Assignment> existingAssignment, Map<String, Set<List<Integer>>> topologyToExecutors, String scratchTopologyId) {
    Map<String, Set<List<Integer>>> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignment.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> allExecutors = topologyToExecutors.get(topoId);
        Set<List<Integer>> aliveExecutors;
        if (topoId.equals(scratchTopologyId)) {
            aliveExecutors = allExecutors;
        } else {
            aliveExecutors = new HashSet<>(aliveExecutors(topoId, allExecutors, assignment));
        }
        ret.put(topoId, aliveExecutors);
    }
    return ret;
}

Example 24 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method getResourcesForTopology.

private TopologyResources getResourcesForTopology(String topoId, StormBase base) throws NotAliveException, AuthorizationException, InvalidTopologyException, IOException {
    TopologyResources ret = idToResources.get().get(topoId);
    if (ret == null) {
        try {
            IStormClusterState state = stormClusterState;
            TopologyDetails details = readTopologyDetails(topoId, base);
            Assignment assignment = state.assignmentInfo(topoId, null);
            ret = new TopologyResources(details, assignment);
        } catch (KeyNotFoundException e) {
            // This can happen when a topology is first coming up
            // It's thrown by the blobstore code
            LOG.error("Failed to get topology details", e);
            ret = new TopologyResources();
        }
    }
    return ret;
}

Example 25 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method lockingMkAssignments.

private void lockingMkAssignments(Map<String, Assignment> existingAssignments, Map<String, StormBase> bases, String scratchTopoId, List<String> assignedTopologyIds, IStormClusterState state, Map<String, TopologyDetails> tds) throws Exception {
    Topologies topologies = new Topologies(tds);
    synchronized (schedLock) {
        Map<String, SchedulerAssignment> newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
        Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeTopoToExecToNodePort(newSchedulerAssignments, assignedTopologyIds);
        Map<String, Map<WorkerSlot, WorkerResources>> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments);
        int nowSecs = Time.currentTimeSecs();
        Map<String, SupervisorDetails> basicSupervisorDetailsMap = basicSupervisorDetailsMap(state);
        // construct the final Assignments by adding start-times etc into it
        Map<String, Assignment> newAssignments = new HashMap<>();
        for (Entry<String, Map<List<Long>, List<Object>>> entry : topologyToExecutorToNodePort.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            if (execToNodePort == null) {
                execToNodePort = new HashMap<>();
            }
            Set<String> allNodes = new HashSet<>();
            for (List<Object> nodePort : execToNodePort.values()) {
                allNodes.add((String) nodePort.get(0));
            }
            Map<String, String> allNodeHost = new HashMap<>();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment != null) {
                allNodeHost.putAll(existingAssignment.get_node_host());
            }
            for (String node : allNodes) {
                String host = inimbus.getHostName(basicSupervisorDetailsMap, node);
                if (host != null) {
                    allNodeHost.put(node, host);
                }
            }
            Map<List<Long>, NodeInfo> execNodeInfo = null;
            if (existingAssignment != null) {
                execNodeInfo = existingAssignment.get_executor_node_port();
            }
            List<List<Long>> reassignExecutors = changedExecutors(execNodeInfo, execToNodePort);
            Map<List<Long>, Long> startTimes = new HashMap<>();
            if (existingAssignment != null) {
                startTimes.putAll(existingAssignment.get_executor_start_time_secs());
            }
            for (List<Long> id : reassignExecutors) {
                startTimes.put(id, (long) nowSecs);
            }
            Map<WorkerSlot, WorkerResources> workerToResources = newAssignedWorkerToResources.get(topoId);
            if (workerToResources == null) {
                workerToResources = new HashMap<>();
            }
            Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR));
            Map<String, String> justAssignedKeys = new HashMap<>(allNodeHost);
            // Modifies justAssignedKeys
            justAssignedKeys.keySet().retainAll(allNodes);
            newAssignment.set_node_host(justAssignedKeys);
            // convert NodePort to NodeInfo (again!!!).
            Map<List<Long>, NodeInfo> execToNodeInfo = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                List<Object> nodePort = execAndNodePort.getValue();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                execToNodeInfo.put(execAndNodePort.getKey(), ni);
            }
            newAssignment.set_executor_node_port(execToNodeInfo);
            newAssignment.set_executor_start_time_secs(startTimes);
            // do another conversion (lets just make this all common)
            Map<NodeInfo, WorkerResources> workerResources = new HashMap<>();
            for (Entry<WorkerSlot, WorkerResources> wr : workerToResources.entrySet()) {
                WorkerSlot nodePort = wr.getKey();
                NodeInfo ni = new NodeInfo();
                ni.set_node(nodePort.getNodeId());
                ni.add_to_port(nodePort.getPort());
                WorkerResources resources = wr.getValue();
                workerResources.put(ni, resources);
            }
            newAssignment.set_worker_resources(workerResources);
            TopologyDetails td = tds.get(topoId);
            newAssignment.set_owner(td.getTopologySubmitter());
            newAssignments.put(topoId, newAssignment);
        }
        boolean assignmentChanged = auditAssignmentChanges(existingAssignments, newAssignments);
        if (assignmentChanged) {
            LOG.debug("RESETTING id->resources and id->worker-resources cache!");
            idToResources.set(new HashMap<>());
            idToWorkerResources.set(new HashMap<>());
        }
        // only log/set when there's been a change to the assignment
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            TopologyDetails td = topologies.getById(topoId);
            if (assignment.equals(existingAssignment)) {
                LOG.debug("Assignment for {} hasn't changed", topoId);
            } else {
                LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment);
                state.setAssignment(topoId, assignment, td.getConf());
            }
        }
        // grouping assignment by node to see the nodes diff, then notify nodes/supervisors to synchronize its owned assignment
        // because the number of existing assignments is small for every scheduling round,
        // we expect to notify supervisors at almost the same time
        Map<String, String> totalAssignmentsChangedNodes = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            totalAssignmentsChangedNodes.putAll(assignmentChangedNodes(existingAssignment, assignment));
        }
        notifySupervisorsAssignments(newAssignments, assignmentsDistributer, totalAssignmentsChangedNodes, basicSupervisorDetailsMap, getMetricsRegistry());
        Map<String, Collection<WorkerSlot>> addedSlots = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment == null) {
                existingAssignment = new Assignment();
                existingAssignment.set_executor_node_port(new HashMap<>());
                existingAssignment.set_executor_start_time_secs(new HashMap<>());
            }
            Set<WorkerSlot> newSlots = newlyAddedSlots(existingAssignment, assignment);
            addedSlots.put(topoId, newSlots);
        }
        inimbus.assignSlots(topologies, addedSlots);
    }
}