Examples with ProfileRequest org.apache.storm.generated.ProfileRequest used on opensource projects

Example 1 with ProfileRequest

use of org.apache.storm.generated.ProfileRequest in project storm by apache.

the class Nimbus method getComponentPendingProfileActions.

@Override
public List<ProfileRequest> getComponentPendingProfileActions(String id, String componentId, ProfileAction action) throws TException {
    try {
        getComponentPendingProfileActionsCalls.mark();
        CommonTopoInfo info = getCommonTopoInfo(id, "getComponentPendingProfileActions");
        Map<String, String> nodeToHost = info.assignment.get_node_host();
        Map<List<? extends Number>, List<Object>> exec2hostPort = new HashMap<>();
        for (Entry<List<Long>, NodeInfo> entry : info.assignment.get_executor_node_port().entrySet()) {
            NodeInfo ni = entry.getValue();
            List<Object> hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue());
            exec2hostPort.put(entry.getKey(), hostPort);
        }
        List<Map<String, Object>> nodeInfos = StatsUtil.extractNodeInfosFromHbForComp(exec2hostPort, info.taskToComponent, false, componentId);
        List<ProfileRequest> ret = new ArrayList<>();
        for (Map<String, Object> ni : nodeInfos) {
            String niHost = (String) ni.get("host");
            int niPort = ((Integer) ni.get("port")).intValue();
            ProfileRequest newestMatch = null;
            long reqTime = -1;
            for (ProfileRequest req : stormClusterState.getTopologyProfileRequests(id)) {
                String expectedHost = req.get_nodeInfo().get_node();
                int expectedPort = req.get_nodeInfo().get_port_iterator().next().intValue();
                ProfileAction expectedAction = req.get_action();
                if (niHost.equals(expectedHost) && niPort == expectedPort && action == expectedAction) {
                    long time = req.get_time_stamp();
                    if (time > reqTime) {
                        reqTime = time;
                        newestMatch = req;
                    }
                }
            }
            if (newestMatch != null) {
                ret.add(newestMatch);
            }
        }
        LOG.info("Latest profile actions for topology {} component {} {}", id, componentId, ret);
        return ret;
    } catch (Exception e) {
        LOG.warn("Get comp actions topology exception. (topology id='{}')", id, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 2 with ProfileRequest

use of org.apache.storm.generated.ProfileRequest in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
        List<String> stormIds = stormClusterState.assignments(syncCallback);
        Map<String, VersionedData<Assignment>> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            //Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (iSuper.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}

Example 3 with ProfileRequest

use of org.apache.storm.generated.ProfileRequest in project storm by apache.

the class SlotTest method testRunWithProfileActions.

@Test
public void testRunWithProfileActions() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        //NOT going to timeout for a while
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100);
        when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
        when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
        ILocalizer localizer = mock(ILocalizer.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state);
        Set<TopoProfileAction> profileActions = new HashSet<>();
        ProfileRequest request = new ProfileRequest();
        request.set_action(ProfileAction.JPROFILE_STOP);
        NodeInfo info = new NodeInfo();
        info.set_node("localhost");
        info.add_to_port(port);
        request.set_nodeInfo(info);
        //3 seconds from now
        request.set_time_stamp(Time.currentTimeMillis() + 3000);
        TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
        profileActions.add(profile);
        Set<TopoProfileAction> expectedPending = new HashSet<>();
        expectedPending.add(profile);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withProfileActions(profileActions, Collections.<TopoProfileAction>emptySet());
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, false);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, true);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 4000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 5000);
    }
}

Example 4 with ProfileRequest

use of org.apache.storm.generated.ProfileRequest in project storm by apache.

the class BasicContainerTest method testRunProfiling.

@Test
public void testRunProfiling() throws Exception {
    final long pid = 100;
    final String topoId = "test_topology";
    final int port = 8080;
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "testing");
    final String topoRoot = ContainerTest.asAbsPath(stormLocal, topoId, String.valueOf(port));
    final File workerArtifactsPid = ContainerTest.asAbsFile(topoRoot, "worker.pid");
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurpString(workerArtifactsPid)).thenReturn(String.valueOf(pid));
    LocalState ls = mock(LocalState.class);
    MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
    //HEAP DUMP
    ProfileRequest req = new ProfileRequest();
    req.set_action(ProfileAction.JMAP_DUMP);
    mc.runProfiling(req, false);
    assertEquals(1, mc.profileCmds.size());
    CommandRun cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "jmap", topoRoot), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
    //JSTACK DUMP
    req.set_action(ProfileAction.JSTACK_DUMP);
    mc.runProfiling(req, false);
    assertEquals(1, mc.profileCmds.size());
    cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "jstack", topoRoot), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
    //RESTART
    req.set_action(ProfileAction.JVM_RESTART);
    mc.runProfiling(req, false);
    assertEquals(1, mc.profileCmds.size());
    cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "kill"), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
    //JPROFILE DUMP
    req.set_action(ProfileAction.JPROFILE_DUMP);
    mc.runProfiling(req, false);
    assertEquals(1, mc.profileCmds.size());
    cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "dump", topoRoot), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
    //JPROFILE START
    req.set_action(ProfileAction.JPROFILE_STOP);
    mc.runProfiling(req, false);
    assertEquals(1, mc.profileCmds.size());
    cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "start"), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
    //JPROFILE STOP
    req.set_action(ProfileAction.JPROFILE_STOP);
    mc.runProfiling(req, true);
    assertEquals(1, mc.profileCmds.size());
    cmd = mc.profileCmds.get(0);
    mc.profileCmds.clear();
    assertEquals(Arrays.asList("profile", String.valueOf(pid), "stop", topoRoot), cmd.cmd);
    assertEquals(new File(topoRoot), cmd.pwd);
}