Search in sources :

Example 1 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class Nimbus method getTopologyInfoWithOpts.

@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyInfoWithOptsCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
        if (common.base == null) {
            throw new NotAliveException(topoId);
        }
        IStormClusterState state = stormClusterState;
        NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
        Map<String, List<ErrorInfo>> errors = new HashMap<>();
        for (String component : common.allComponents) {
            switch(numErrChoice) {
                case NONE:
                    errors.put(component, Collections.emptyList());
                    break;
                case ONE:
                    List<ErrorInfo> errList = new ArrayList<>();
                    ErrorInfo info = state.lastError(topoId, component);
                    if (info != null) {
                        errList.add(info);
                    }
                    errors.put(component, errList);
                    break;
                case ALL:
                    errors.put(component, state.errors(topoId, component));
                    break;
                default:
                    LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                    errors.put(component, state.errors(topoId, component));
                    break;
            }
        }
        List<ExecutorSummary> summaries = new ArrayList<>();
        if (common.assignment != null) {
            for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                ExecutorInfo execInfo = toExecInfo(entry.getKey());
                String host = entry.getValue().get_node();
                Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
                if (heartbeat == null) {
                    heartbeat = Collections.emptyMap();
                }
                ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
                //heartbeats "stats"
                Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
                if (hb != null) {
                    Map ex = (Map) hb.get("stats");
                    if (ex != null) {
                        ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                        summ.set_stats(stats);
                    }
                }
                summaries.add(summ);
            }
        }
        TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
        if (common.base.is_set_owner()) {
            topoInfo.set_owner(common.base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, common.base);
        if (resources != null) {
            topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoInfo.set_requested_cpu(resources.getRequestedCpu());
            topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        if (common.base.is_set_component_debug()) {
            topoInfo.set_component_debug(common.base.get_component_debug());
        }
        topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        return topoInfo;
    } catch (Exception e) {
        LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) ExecutorStats(org.apache.storm.generated.ExecutorStats) ArrayList(java.util.ArrayList) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) NotAliveException(org.apache.storm.generated.NotAliveException) ExecutorInfo(org.apache.storm.generated.ExecutorInfo) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) ErrorInfo(org.apache.storm.generated.ErrorInfo) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) NodeInfo(org.apache.storm.generated.NodeInfo) NumErrorsChoice(org.apache.storm.generated.NumErrorsChoice) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) TopologyInfo(org.apache.storm.generated.TopologyInfo)

Example 2 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class InOrderDeliveryTest method printMetrics.

public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
    TopologyInfo info = client.getTopologyInfoByName(name);
    int uptime = info.get_uptime_secs();
    long acked = 0;
    long failed = 0;
    double weightedAvgTotal = 0.0;
    for (ExecutorSummary exec : info.get_executors()) {
        if ("spout".equals(exec.get_component_id())) {
            SpoutStats stats = exec.get_stats().get_specific().get_spout();
            Map<String, Long> failedMap = stats.get_failed().get(":all-time");
            Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
            Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
            for (String key : ackedMap.keySet()) {
                if (failedMap != null) {
                    Long tmp = failedMap.get(key);
                    if (tmp != null) {
                        failed += tmp;
                    }
                }
                long ackVal = ackedMap.get(key);
                double latVal = avgLatMap.get(key) * ackVal;
                acked += ackVal;
                weightedAvgTotal += latVal;
            }
        }
    }
    double avgLatency = weightedAvgTotal / acked;
    System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " + (((double) acked) / uptime + " failed: " + failed));
}
Also used : SpoutStats(org.apache.storm.generated.SpoutStats) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 3 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class TopoWrap method getComponentExecutorCount.

public long getComponentExecutorCount(final String componentId) throws TException {
    TopologyInfo info = getInfo();
    List<ExecutorSummary> executors = info.get_executors();
    return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).count();
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) NumberFormat(java.text.NumberFormat) ArrayList(java.util.ArrayList) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) HashSet(java.util.HashSet) AssertUtil(org.apache.storm.st.utils.AssertUtil) Assert(org.testng.Assert) StormTopology(org.apache.storm.generated.StormTopology) Locale(java.util.Locale) Map(java.util.Map) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TimeUtil(org.apache.storm.st.utils.TimeUtil) StormSubmitter(org.apache.storm.StormSubmitter) Logger(org.slf4j.Logger) ExecutorStats(org.apache.storm.generated.ExecutorStats) ImmutableMap(com.google.common.collect.ImmutableMap) MalformedURLException(java.net.MalformedURLException) ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) StringDecorator(org.apache.storm.st.utils.StringDecorator) Collection(java.util.Collection) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) Set(java.util.Set) ExecutorAggregateStats(org.apache.storm.generated.ExecutorAggregateStats) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) TopologyInfo(org.apache.storm.generated.TopologyInfo) FromJson(org.apache.storm.st.topology.window.data.FromJson) Utils(org.apache.storm.utils.Utils) Collectors(java.util.stream.Collectors) TException(org.apache.storm.thrift.TException) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) IOUtils(org.apache.commons.io.IOUtils) AuthorizationException(org.apache.storm.generated.AuthorizationException) URLEncoder(java.net.URLEncoder) List(java.util.List) Config(org.apache.storm.Config) Pattern(java.util.regex.Pattern) TopologySummary(org.apache.storm.generated.TopologySummary) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 4 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class TopoWrap method getAllTimeEmittedCount.

public long getAllTimeEmittedCount(final String componentId) throws TException {
    TopologyInfo info = getInfo();
    final List<ExecutorSummary> executors = info.get_executors();
    return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).mapToLong(summary -> {
        ExecutorStats executorStats = summary.get_stats();
        if (executorStats == null) {
            return 0L;
        }
        Map<String, Map<String, Long>> emitted = executorStats.get_emitted();
        if (emitted == null) {
            return 0L;
        }
        Map<String, Long> allTime = emitted.get(":all-time");
        if (allTime == null) {
            return 0L;
        }
        return allTime.get(Utils.DEFAULT_STREAM_ID);
    }).sum();
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) NumberFormat(java.text.NumberFormat) ArrayList(java.util.ArrayList) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) HashSet(java.util.HashSet) AssertUtil(org.apache.storm.st.utils.AssertUtil) Assert(org.testng.Assert) StormTopology(org.apache.storm.generated.StormTopology) Locale(java.util.Locale) Map(java.util.Map) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TimeUtil(org.apache.storm.st.utils.TimeUtil) StormSubmitter(org.apache.storm.StormSubmitter) Logger(org.slf4j.Logger) ExecutorStats(org.apache.storm.generated.ExecutorStats) ImmutableMap(com.google.common.collect.ImmutableMap) MalformedURLException(java.net.MalformedURLException) ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) StringDecorator(org.apache.storm.st.utils.StringDecorator) Collection(java.util.Collection) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) Set(java.util.Set) ExecutorAggregateStats(org.apache.storm.generated.ExecutorAggregateStats) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) TopologyInfo(org.apache.storm.generated.TopologyInfo) FromJson(org.apache.storm.st.topology.window.data.FromJson) Utils(org.apache.storm.utils.Utils) Collectors(java.util.stream.Collectors) TException(org.apache.storm.thrift.TException) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) IOUtils(org.apache.commons.io.IOUtils) AuthorizationException(org.apache.storm.generated.AuthorizationException) URLEncoder(java.net.URLEncoder) List(java.util.List) Config(org.apache.storm.Config) Pattern(java.util.regex.Pattern) TopologySummary(org.apache.storm.generated.TopologySummary) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) ExecutorStats(org.apache.storm.generated.ExecutorStats) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 5 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class TopoWrap method getLogUrls.

/**
 * Get the Logviewer worker log URLs for the specified component.
 */
public Set<ExecutorURL> getLogUrls(final String componentId) throws TException, MalformedURLException {
    ComponentPageInfo componentPageInfo = cluster.getNimbusClient().getComponentPageInfo(id, componentId, null, false);
    List<ExecutorAggregateStats> executorStats = componentPageInfo.get_exec_stats();
    Set<ExecutorURL> urls = new HashSet<>();
    for (ExecutorAggregateStats execStat : executorStats) {
        ExecutorSummary execSummary = execStat.get_exec_summary();
        String host = execSummary.get_host();
        int executorPort = execSummary.get_port();
        // http://supervisor2:8000/download/DemoTest-26-1462229009%2F6703%2Fworker.log
        // http://supervisor2:8000/log?file=SlidingWindowCountTest-9-1462388349%2F6703%2Fworker.log
        int logViewerPort = 8000;
        ExecutorURL executorURL = new ExecutorURL(componentId, host, logViewerPort, executorPort, id);
        urls.add(executorURL);
    }
    return urls;
}
Also used : ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) ExecutorAggregateStats(org.apache.storm.generated.ExecutorAggregateStats) HashSet(java.util.HashSet) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Aggregations

ExecutorSummary (org.apache.storm.generated.ExecutorSummary)24 HashMap (java.util.HashMap)14 ArrayList (java.util.ArrayList)13 TopologyInfo (org.apache.storm.generated.TopologyInfo)13 Map (java.util.Map)12 List (java.util.List)9 ExecutorStats (org.apache.storm.generated.ExecutorStats)8 HashSet (java.util.HashSet)6 SpoutStats (org.apache.storm.generated.SpoutStats)6 ExecutorInfo (org.apache.storm.generated.ExecutorInfo)5 TopologySummary (org.apache.storm.generated.TopologySummary)5 ImmutableMap (com.google.common.collect.ImmutableMap)4 File (java.io.File)4 Collectors (java.util.stream.Collectors)4 Config (org.apache.storm.Config)4 ExecutorAggregateStats (org.apache.storm.generated.ExecutorAggregateStats)4 StormTopology (org.apache.storm.generated.StormTopology)4 Logger (org.slf4j.Logger)4 LoggerFactory (org.slf4j.LoggerFactory)4 IOException (java.io.IOException)3