Examples with Assignment org.apache.storm.generated.Assignment used on opensource projects

Example 6 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method getComponentPageInfo.

@Override
public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getComponentPageInfoCalls.mark();
        CommonTopoInfo info = getCommonTopoInfo(topoId, "getComponentPageInfo");
        if (info.base == null) {
            throw new WrappedNotAliveException(topoId);
        }
        StormTopology topology = info.topology;
        Map<String, Object> topoConf = info.topoConf;
        topoConf = Utils.merge(conf, topoConf);
        Assignment assignment = info.assignment;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        Map<String, String> nodeToHost;
        Map<List<Long>, List<Object>> exec2HostPort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                List<Object> hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
                exec2HostPort.put(entry.getKey(), hostPort);
            }
        } else {
            nodeToHost = Collections.emptyMap();
        }
        ComponentPageInfo compPageInfo = StatsUtil.aggCompExecsStats(exec2HostPort, info.taskToComponent, info.beats, window, includeSys, topoId, topology, componentId);
        if (compPageInfo.get_component_type() == ComponentType.SPOUT) {
            NormalizedResourceRequest spoutResources = ResourceUtils.getSpoutResources(topology, topoConf, componentId);
            if (spoutResources == null) {
                spoutResources = new NormalizedResourceRequest(topoConf, componentId);
            }
            compPageInfo.set_resources_map(spoutResources.toNormalizedMap());
        } else {
            // bolt
            NormalizedResourceRequest boltResources = ResourceUtils.getBoltResources(topology, topoConf, componentId);
            if (boltResources == null) {
                boltResources = new NormalizedResourceRequest(topoConf, componentId);
            }
            compPageInfo.set_resources_map(boltResources.toNormalizedMap());
        }
        compPageInfo.set_topology_name(info.topoName);
        compPageInfo.set_errors(stormClusterState.errors(topoId, componentId));
        compPageInfo.set_topology_status(extractStatusStr(info.base));
        if (info.base.is_set_component_debug()) {
            DebugOptions debug = info.base.get_component_debug().get(componentId);
            if (debug != null) {
                compPageInfo.set_debug_options(debug);
            }
        }
        // Add the event logger details.
        Map<String, List<Integer>> compToTasks = Utils.reverseMap(info.taskToComponent);
        if (compToTasks.containsKey(StormCommon.EVENTLOGGER_COMPONENT_ID)) {
            List<Integer> tasks = compToTasks.get(StormCommon.EVENTLOGGER_COMPONENT_ID);
            tasks.sort(null);
            // Find the task the events from this component route to.
            int taskIndex = TupleUtils.chooseTaskIndex(Collections.singletonList(componentId), tasks.size());
            int taskId = tasks.get(taskIndex);
            String host = null;
            Integer port = null;
            for (Entry<List<Long>, List<Object>> entry : exec2HostPort.entrySet()) {
                int start = entry.getKey().get(0).intValue();
                int end = entry.getKey().get(1).intValue();
                if (taskId >= start && taskId <= end) {
                    host = (String) entry.getValue().get(0);
                    port = ((Number) entry.getValue().get(1)).intValue();
                    break;
                }
            }
            if (host != null && port != null) {
                compPageInfo.set_eventlog_host(host);
                compPageInfo.set_eventlog_port(port);
            }
        }
        return compPageInfo;
    } catch (Exception e) {
        LOG.warn("getComponentPageInfo exception. (topo id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 7 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method computeSupervisorToDeadPorts.

private Map<String, Set<Long>> computeSupervisorToDeadPorts(Map<String, Assignment> existingAssignments, Map<String, Set<List<Integer>>> topologyToExecutors, Map<String, Set<List<Integer>>> topologyToAliveExecutors) {
    Map<String, Set<Long>> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> allExecutors = topologyToExecutors.get(topoId);
        Set<List<Integer>> aliveExecutors = topologyToAliveExecutors.get(topoId);
        Set<List<Integer>> deadExecutors = new HashSet<>(allExecutors);
        deadExecutors.removeAll(aliveExecutors);
        Map<List<Long>, NodeInfo> execToNodePort = assignment.get_executor_node_port();
        for (Entry<List<Long>, NodeInfo> assigned : execToNodePort.entrySet()) {
            if (deadExecutors.contains(asIntExec(assigned.getKey()))) {
                NodeInfo info = assigned.getValue();
                String superId = info.get_node();
                Set<Long> ports = ret.get(superId);
                if (ports == null) {
                    ports = new HashSet<>();
                    ret.put(superId, ports);
                }
                ports.addAll(info.get_port());
            }
        }
    }
    return ret;
}

Example 8 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method mkAssignments.

private void mkAssignments(String scratchTopoId) throws Exception {
    try {
        if (!isReadyForMKAssignments()) {
            return;
        }
        // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {}
        // filter out ones which have a executor timeout
        // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors
        // should be in each slot (e.g., 4, 4, 4, 5)
        // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots
        // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be
        // reassigned to. worst comes to worse the executor will timeout and won't assign here next time around
        IStormClusterState state = stormClusterState;
        // read all the topologies
        Map<String, StormBase> bases;
        Map<String, TopologyDetails> tds = new HashMap<>();
        synchronized (submitLock) {
            // should promote: only fetch storm bases of topologies that need scheduling.
            bases = state.topologyBases();
            for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
                Entry<String, StormBase> entry = it.next();
                String id = entry.getKey();
                StormBase base = entry.getValue();
                try {
                    tds.put(id, readTopologyDetails(id, base));
                } catch (KeyNotFoundException e) {
                    // A race happened and it is probably not running
                    it.remove();
                }
            }
        }
        List<String> assignedTopologyIds = state.assignments(null);
        Map<String, Assignment> existingAssignments = new HashMap<>();
        for (String id : assignedTopologyIds) {
            // will be treated as free slot in the scheduler code.
            if (!id.equals(scratchTopoId)) {
                Assignment currentAssignment = state.assignmentInfo(id, null);
                if (!currentAssignment.is_set_owner()) {
                    TopologyDetails td = tds.get(id);
                    if (td != null) {
                        currentAssignment.set_owner(td.getTopologySubmitter());
                        state.setAssignment(id, currentAssignment, td.getConf());
                    }
                }
                existingAssignments.put(id, currentAssignment);
            }
        }
        // make the new assignments for topologies
        lockingMkAssignments(existingAssignments, bases, scratchTopoId, assignedTopologyIds, state, tds);
    } catch (Exception e) {
        this.mkAssignmentsErrors.mark();
        throw e;
    }
}

Example 9 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class ReadClusterState method readAssignments.

protected Map<Integer, LocalAssignment> readAssignments(Map<String, Assignment> assignmentsSnapshot) {
    try {
        Map<Integer, LocalAssignment> portLocalAssignment = new HashMap<>();
        for (Map.Entry<String, Assignment> assignEntry : assignmentsSnapshot.entrySet()) {
            String topoId = assignEntry.getKey();
            Assignment assignment = assignEntry.getValue();
            Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
            for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
                Integer port = entry.getKey();
                LocalAssignment la = entry.getValue();
                if (!portLocalAssignment.containsKey(port)) {
                    portLocalAssignment.put(port, la);
                } else {
                    throw new RuntimeException("Should not have multiple topologies assigned to one port " + port + " " + la + " " + portLocalAssignment);
                }
            }
        }
        readRetry.set(0);
        return portLocalAssignment;
    } catch (RuntimeException e) {
        if (readRetry.get() > 2) {
            throw e;
        } else {
            readRetry.addAndGet(1);
        }
        LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
        return null;
    }
}

Example 10 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        List<String> stormIds = stormClusterState.assignments(null);
        Map<String, Assignment> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            // Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (supervisor.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        supervisor.assigned(allPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}