Examples with NodeInfo org.apache.storm.generated.NodeInfo used on opensource projects

Example 11 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class Nimbus method getComponentPendingProfileActions.

@Override
public List<ProfileRequest> getComponentPendingProfileActions(String id, String componentId, ProfileAction action) throws TException {
    try {
        getComponentPendingProfileActionsCalls.mark();
        CommonTopoInfo info = getCommonTopoInfo(id, "getComponentPendingProfileActions");
        Map<List<? extends Number>, List<Object>> exec2hostPort = new HashMap<>();
        if (info.assignment != null) {
            Map<String, String> nodeToHost = info.assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : info.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue());
                exec2hostPort.put(entry.getKey(), hostPort);
            }
        }
        List<Map<String, Object>> nodeInfos = StatsUtil.extractNodeInfosFromHbForComp(exec2hostPort, info.taskToComponent, false, componentId);
        List<ProfileRequest> ret = new ArrayList<>();
        for (Map<String, Object> ni : nodeInfos) {
            String niHost = (String) ni.get("host");
            int niPort = ((Integer) ni.get("port")).intValue();
            ProfileRequest newestMatch = null;
            long reqTime = -1;
            for (ProfileRequest req : stormClusterState.getTopologyProfileRequests(id)) {
                String expectedHost = req.get_nodeInfo().get_node();
                int expectedPort = req.get_nodeInfo().get_port_iterator().next().intValue();
                ProfileAction expectedAction = req.get_action();
                if (niHost.equals(expectedHost) && niPort == expectedPort && action == expectedAction) {
                    long time = req.get_time_stamp();
                    if (time > reqTime) {
                        reqTime = time;
                        newestMatch = req;
                    }
                }
            }
            if (newestMatch != null) {
                ret.add(newestMatch);
            }
        }
        LOG.info("Latest profile actions for topology {} component {} {}", id, componentId, ret);
        return ret;
    } catch (Exception e) {
        LOG.warn("Get comp actions topology exception. (topology id='{}')", id, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 12 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        List<String> stormIds = stormClusterState.assignments(null);
        Map<String, Assignment> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            // Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (supervisor.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        supervisor.assigned(allPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}

Example 13 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class Nimbus method getSupervisorPageInfo.

@Override
public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getSupervisorPageInfoCalls.mark();
        IStormClusterState state = stormClusterState;
        Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
        Map<String, List<String>> hostToSuperId = new HashMap<>();
        for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
            String h = entry.getValue().get_hostname();
            List<String> superIds = hostToSuperId.get(h);
            if (superIds == null) {
                superIds = new ArrayList<>();
                hostToSuperId.put(h, superIds);
            }
            superIds.add(entry.getKey());
        }
        List<String> supervisorIds = null;
        if (superId == null) {
            supervisorIds = hostToSuperId.get(host);
        } else {
            supervisorIds = Arrays.asList(superId);
        }
        SupervisorPageInfo pageInfo = new SupervisorPageInfo();
        Map<String, Assignment> topoToAssignment = state.assignmentsInfo();
        for (String sid : supervisorIds) {
            SupervisorInfo info = superInfos.get(sid);
            LOG.info("SIDL {} SI: {} ALL: {}", sid, info, superInfos);
            SupervisorSummary supSum = makeSupervisorSummary(sid, info);
            pageInfo.add_to_supervisor_summaries(supSum);
            List<String> superTopologies = topologiesOnSupervisor(topoToAssignment, sid);
            Set<String> userTopologies = filterAuthorized("getTopology", superTopologies);
            for (String topoId : superTopologies) {
                CommonTopoInfo common = getCommonTopoInfo(topoId, "getSupervisorPageInfo");
                String topoName = common.topoName;
                Assignment assignment = common.assignment;
                Map<List<Integer>, Map<String, Object>> beats = common.beats;
                Map<Integer, String> taskToComp = common.taskToComponent;
                Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
                Map<String, String> nodeToHost;
                if (assignment != null) {
                    Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
                    for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                        NodeInfo ni = entry.getValue();
                        List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                        exec2NodePort.put(entry.getKey(), nodePort);
                    }
                    nodeToHost = assignment.get_node_host();
                } else {
                    nodeToHost = Collections.emptyMap();
                }
                Map<WorkerSlot, WorkerResources> workerResources = getWorkerResourcesForTopology(topoId);
                boolean isAllowed = userTopologies.contains(topoId);
                String owner = (common.base == null) ? null : common.base.get_owner();
                for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid, owner)) {
                    pageInfo.add_to_worker_summaries(workerSummary);
                }
            }
        }
        return pageInfo;
    } catch (Exception e) {
        LOG.warn("Get super page info exception. (super id='{}')", superId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 14 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class LocalAssignmentsBackendTest method mockedAssignment.

private Assignment mockedAssignment(int i) {
    Assignment ass = new Assignment();
    ass.set_master_code_dir("master_code_dir" + i);
    HashMap node_to_host = new HashMap();
    node_to_host.put("node" + i, "host" + i);
    ass.set_node_host(node_to_host);
    Map<List<Long>, NodeInfo> executor_node_port = new HashMap<>();
    Set<Long> nodePorts = new HashSet<>();
    nodePorts.add(9723L);
    executor_node_port.put(Arrays.asList(i + 0L), new NodeInfo("node" + i, nodePorts));
    ass.set_executor_node_port(executor_node_port);
    Map<List<Long>, Long> executor_start_time_secs = new HashMap<>();
    executor_start_time_secs.put(Arrays.asList(1L), 12345L);
    ass.set_executor_start_time_secs(executor_start_time_secs);
    ass.set_worker_resources(new HashedMap());
    ass.set_total_shared_off_heap(new HashedMap());
    ass.set_owner("o");
    return ass;
}

Example 15 with NodeInfo

use of org.apache.storm.generated.NodeInfo in project storm by apache.

the class SlotTest method testRunWithProfileActions.

@Test
public void testRunWithProfileActions() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        // NOT going to timeout for a while
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100);
        when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
        when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, new SlotMetrics(new StormMetricsRegistry()));
        Set<TopoProfileAction> profileActions = new HashSet<>();
        ProfileRequest request = new ProfileRequest();
        request.set_action(ProfileAction.JPROFILE_STOP);
        NodeInfo info = new NodeInfo();
        info.set_node("localhost");
        info.add_to_port(port);
        request.set_nodeInfo(info);
        // 3 seconds from now
        request.set_time_stamp(Time.currentTimeMillis() + 3000);
        TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
        profileActions.add(profile);
        Set<TopoProfileAction> expectedPending = new HashSet<>();
        expectedPending.add(profile);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, slotMetrics).withProfileActions(profileActions, Collections.<TopoProfileAction>emptySet());
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, false);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, true);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 4000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 5000);
    }
}