Examples with ExecutorInfo org.apache.storm.generated.ExecutorInfo used on opensource projects

Example 11 with ExecutorInfo

use of org.apache.storm.generated.ExecutorInfo in project storm by apache.

the class StatsUtil method thriftifyZkWorkerHb.

// =====================================================================================
// thriftify stats methods
// =====================================================================================
public static ClusterWorkerHeartbeat thriftifyZkWorkerHb(Map<String, Object> heartbeat) {
    ClusterWorkerHeartbeat ret = new ClusterWorkerHeartbeat();
    ret.set_uptime_secs(getByKeyOr0(heartbeat, UPTIME).intValue());
    ret.set_storm_id((String) getByKey(heartbeat, "storm-id"));
    ret.set_time_secs(getByKeyOr0(heartbeat, TIME_SECS).intValue());
    Map<ExecutorInfo, ExecutorStats> convertedStats = new HashMap<>();
    Map<List<Integer>, ExecutorStats> executorStats = getMapByKey(heartbeat, EXECUTOR_STATS);
    if (executorStats != null) {
        for (Map.Entry<List<Integer>, ExecutorStats> entry : executorStats.entrySet()) {
            List<Integer> executor = entry.getKey();
            ExecutorStats stats = entry.getValue();
            if (null != stats) {
                convertedStats.put(new ExecutorInfo(executor.get(0), executor.get(1)), stats);
            }
        }
    }
    ret.set_executor_stats(convertedStats);
    return ret;
}

Example 12 with ExecutorInfo

use of org.apache.storm.generated.ExecutorInfo in project storm by apache.

the class Nimbus method getTopologyInfoWithOpts.

@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyInfoWithOptsCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
        if (common.base == null) {
            throw new NotAliveException(topoId);
        }
        IStormClusterState state = stormClusterState;
        NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
        Map<String, List<ErrorInfo>> errors = new HashMap<>();
        for (String component : common.allComponents) {
            switch(numErrChoice) {
                case NONE:
                    errors.put(component, Collections.emptyList());
                    break;
                case ONE:
                    List<ErrorInfo> errList = new ArrayList<>();
                    ErrorInfo info = state.lastError(topoId, component);
                    if (info != null) {
                        errList.add(info);
                    }
                    errors.put(component, errList);
                    break;
                case ALL:
                    errors.put(component, state.errors(topoId, component));
                    break;
                default:
                    LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                    errors.put(component, state.errors(topoId, component));
                    break;
            }
        }
        List<ExecutorSummary> summaries = new ArrayList<>();
        if (common.assignment != null) {
            for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                ExecutorInfo execInfo = toExecInfo(entry.getKey());
                String host = entry.getValue().get_node();
                Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
                if (heartbeat == null) {
                    heartbeat = Collections.emptyMap();
                }
                ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
                //heartbeats "stats"
                Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
                if (hb != null) {
                    Map ex = (Map) hb.get("stats");
                    if (ex != null) {
                        ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                        summ.set_stats(stats);
                    }
                }
                summaries.add(summ);
            }
        }
        TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
        if (common.base.is_set_owner()) {
            topoInfo.set_owner(common.base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, common.base);
        if (resources != null) {
            topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoInfo.set_requested_cpu(resources.getRequestedCpu());
            topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        if (common.base.is_set_component_debug()) {
            topoInfo.set_component_debug(common.base.get_component_debug());
        }
        topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        return topoInfo;
    } catch (Exception e) {
        LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 13 with ExecutorInfo

use of org.apache.storm.generated.ExecutorInfo in project storm by apache.

the class Worker method doHeartBeat.

public void doHeartBeat() throws IOException {
    LocalState state = ConfigUtils.workerState(workerState.conf, workerState.workerId);
    state.setWorkerHeartBeat(new LSWorkerHeartbeat(Time.currentTimeSecs(), workerState.topologyId, workerState.executors.stream().map(executor -> new ExecutorInfo(executor.get(0).intValue(), executor.get(1).intValue())).collect(Collectors.toList()), workerState.port));
    // this is just in case supervisor is down so that disk doesn't fill up.
    state.cleanup(60);
// it shouldn't take supervisor 120 seconds between listing dir and reading it
}

Example 14 with ExecutorInfo

use of org.apache.storm.generated.ExecutorInfo in project storm by apache.

the class ReadClusterState method readMyExecutors.

protected Map<Integer, LocalAssignment> readMyExecutors(String stormId, String assignmentId, Assignment assignment) {
    Map<Integer, LocalAssignment> portTasks = new HashMap<>();
    Map<Long, WorkerResources> slotsResources = new HashMap<>();
    Map<NodeInfo, WorkerResources> nodeInfoWorkerResourcesMap = assignment.get_worker_resources();
    if (nodeInfoWorkerResourcesMap != null) {
        for (Map.Entry<NodeInfo, WorkerResources> entry : nodeInfoWorkerResourcesMap.entrySet()) {
            if (entry.getKey().get_node().equals(assignmentId)) {
                Set<Long> ports = entry.getKey().get_port();
                for (Long port : ports) {
                    slotsResources.put(port, entry.getValue());
                }
            }
        }
    }
    Map<List<Long>, NodeInfo> executorNodePort = assignment.get_executor_node_port();
    if (executorNodePort != null) {
        for (Map.Entry<List<Long>, NodeInfo> entry : executorNodePort.entrySet()) {
            if (entry.getValue().get_node().equals(assignmentId)) {
                for (Long port : entry.getValue().get_port()) {
                    LocalAssignment localAssignment = portTasks.get(port.intValue());
                    if (localAssignment == null) {
                        List<ExecutorInfo> executors = new ArrayList<>();
                        localAssignment = new LocalAssignment(stormId, executors);
                        if (slotsResources.containsKey(port)) {
                            localAssignment.set_resources(slotsResources.get(port));
                        }
                        portTasks.put(port.intValue(), localAssignment);
                    }
                    List<ExecutorInfo> executorInfoList = localAssignment.get_executors();
                    executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(), entry.getKey().get(entry.getKey().size() - 1).intValue()));
                }
            }
        }
    }
    return portTasks;
}

Example 15 with ExecutorInfo

use of org.apache.storm.generated.ExecutorInfo in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadBaseTopologyBlobs.

@Test
public void testRequestDownloadBaseTopologyBlobs() throws Exception {
    final String topoId = "TOPO";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    ExecutorInfo ei = new ExecutorInfo();
    ei.set_task_start(1);
    ei.set_task_end(1);
    la.add_to_executors(ei);
    final int port = 8080;
    final String jarKey = topoId + "-stormjar.jar";
    final String codeKey = topoId + "-stormcode.ser";
    final String confKey = topoId + "-stormconf.ser";
    final String stormLocal = "/tmp/storm-local/";
    final String stormRoot = stormLocal + topoId + "/";
    final File fStormRoot = new File(stormRoot);
    ClientBlobStore blobStore = mock(ClientBlobStore.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName());
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    conf.put(Config.STORM_LOCAL_DIR, stormLocal);
    Localizer localizer = mock(Localizer.class);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    ConfigUtils mockedCU = mock(ConfigUtils.class);
    Utils mockedU = mock(Utils.class);
    Map<String, Object> topoConf = new HashMap<>(conf);
    AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
    ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
    Utils origUtils = Utils.setInstance(mockedU);
    try {
        when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
        when(mockedCU.supervisorLocalDirImpl(conf)).thenReturn(stormLocal);
        when(mockedU.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore);
        when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        Future<Void> f = al.requestDownloadBaseTopologyBlobs(la, port);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(blobStore).prepare(conf);
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(jarKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(codeKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(confKey), startsWith(stormLocal), eq(blobStore));
        verify(blobStore).shutdown();
        //Extracting the dir from the jar
        verify(mockedU).extractDirFromJarImpl(endsWith("stormjar.jar"), eq("resources"), any(File.class));
        verify(ops).moveDirectoryPreferAtomic(any(File.class), eq(fStormRoot));
        verify(ops).setupStormCodeDir(topoConf, fStormRoot);
        verify(ops, never()).deleteIfExists(any(File.class));
    } finally {
        al.shutdown();
        ConfigUtils.setInstance(orig);
        Utils.setInstance(origUtils);
    }
}