Search in sources :

Example 1 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class Nimbus method getTopologyInfoWithOpts.

@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyInfoWithOptsCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
        if (common.base == null) {
            throw new NotAliveException(topoId);
        }
        IStormClusterState state = stormClusterState;
        NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
        Map<String, List<ErrorInfo>> errors = new HashMap<>();
        for (String component : common.allComponents) {
            switch(numErrChoice) {
                case NONE:
                    errors.put(component, Collections.emptyList());
                    break;
                case ONE:
                    List<ErrorInfo> errList = new ArrayList<>();
                    ErrorInfo info = state.lastError(topoId, component);
                    if (info != null) {
                        errList.add(info);
                    }
                    errors.put(component, errList);
                    break;
                case ALL:
                    errors.put(component, state.errors(topoId, component));
                    break;
                default:
                    LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                    errors.put(component, state.errors(topoId, component));
                    break;
            }
        }
        List<ExecutorSummary> summaries = new ArrayList<>();
        if (common.assignment != null) {
            for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                ExecutorInfo execInfo = toExecInfo(entry.getKey());
                String host = entry.getValue().get_node();
                Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
                if (heartbeat == null) {
                    heartbeat = Collections.emptyMap();
                }
                ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
                //heartbeats "stats"
                Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
                if (hb != null) {
                    Map ex = (Map) hb.get("stats");
                    if (ex != null) {
                        ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                        summ.set_stats(stats);
                    }
                }
                summaries.add(summ);
            }
        }
        TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
        if (common.base.is_set_owner()) {
            topoInfo.set_owner(common.base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, common.base);
        if (resources != null) {
            topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoInfo.set_requested_cpu(resources.getRequestedCpu());
            topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        if (common.base.is_set_component_debug()) {
            topoInfo.set_component_debug(common.base.get_component_debug());
        }
        topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        return topoInfo;
    } catch (Exception e) {
        LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) ExecutorStats(org.apache.storm.generated.ExecutorStats) ArrayList(java.util.ArrayList) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) NotAliveException(org.apache.storm.generated.NotAliveException) ExecutorInfo(org.apache.storm.generated.ExecutorInfo) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) ErrorInfo(org.apache.storm.generated.ErrorInfo) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) NodeInfo(org.apache.storm.generated.NodeInfo) NumErrorsChoice(org.apache.storm.generated.NumErrorsChoice) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) TopologyInfo(org.apache.storm.generated.TopologyInfo)

Example 2 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class InOrderDeliveryTest method printMetrics.

public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
    TopologyInfo info = client.getTopologyInfoByName(name);
    int uptime = info.get_uptime_secs();
    long acked = 0;
    long failed = 0;
    double weightedAvgTotal = 0.0;
    for (ExecutorSummary exec : info.get_executors()) {
        if ("spout".equals(exec.get_component_id())) {
            SpoutStats stats = exec.get_stats().get_specific().get_spout();
            Map<String, Long> failedMap = stats.get_failed().get(":all-time");
            Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
            Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
            for (String key : ackedMap.keySet()) {
                if (failedMap != null) {
                    Long tmp = failedMap.get(key);
                    if (tmp != null) {
                        failed += tmp;
                    }
                }
                long ackVal = ackedMap.get(key);
                double latVal = avgLatMap.get(key) * ackVal;
                acked += ackVal;
                weightedAvgTotal += latVal;
            }
        }
    }
    double avgLatency = weightedAvgTotal / acked;
    System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " + (((double) acked) / uptime + " failed: " + failed));
}
Also used : SpoutStats(org.apache.storm.generated.SpoutStats) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 3 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class TopoWrap method getComponentExecutorCount.

public long getComponentExecutorCount(final String componentId) throws TException {
    TopologyInfo info = getInfo();
    List<ExecutorSummary> executors = info.get_executors();
    return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).count();
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) NumberFormat(java.text.NumberFormat) ArrayList(java.util.ArrayList) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) HashSet(java.util.HashSet) AssertUtil(org.apache.storm.st.utils.AssertUtil) Assert(org.testng.Assert) StormTopology(org.apache.storm.generated.StormTopology) Locale(java.util.Locale) Map(java.util.Map) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TimeUtil(org.apache.storm.st.utils.TimeUtil) StormSubmitter(org.apache.storm.StormSubmitter) Logger(org.slf4j.Logger) ExecutorStats(org.apache.storm.generated.ExecutorStats) ImmutableMap(com.google.common.collect.ImmutableMap) MalformedURLException(java.net.MalformedURLException) ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) StringDecorator(org.apache.storm.st.utils.StringDecorator) Collection(java.util.Collection) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) Set(java.util.Set) ExecutorAggregateStats(org.apache.storm.generated.ExecutorAggregateStats) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) TopologyInfo(org.apache.storm.generated.TopologyInfo) FromJson(org.apache.storm.st.topology.window.data.FromJson) Utils(org.apache.storm.utils.Utils) Collectors(java.util.stream.Collectors) TException(org.apache.storm.thrift.TException) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) IOUtils(org.apache.commons.io.IOUtils) AuthorizationException(org.apache.storm.generated.AuthorizationException) URLEncoder(java.net.URLEncoder) List(java.util.List) Config(org.apache.storm.Config) Pattern(java.util.regex.Pattern) TopologySummary(org.apache.storm.generated.TopologySummary) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 4 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class TopoWrap method getAllTimeEmittedCount.

public long getAllTimeEmittedCount(final String componentId) throws TException {
    TopologyInfo info = getInfo();
    final List<ExecutorSummary> executors = info.get_executors();
    return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).mapToLong(summary -> {
        ExecutorStats executorStats = summary.get_stats();
        if (executorStats == null) {
            return 0L;
        }
        Map<String, Map<String, Long>> emitted = executorStats.get_emitted();
        if (emitted == null) {
            return 0L;
        }
        Map<String, Long> allTime = emitted.get(":all-time");
        if (allTime == null) {
            return 0L;
        }
        return allTime.get(Utils.DEFAULT_STREAM_ID);
    }).sum();
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) URL(java.net.URL) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) NumberFormat(java.text.NumberFormat) ArrayList(java.util.ArrayList) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) HashSet(java.util.HashSet) AssertUtil(org.apache.storm.st.utils.AssertUtil) Assert(org.testng.Assert) StormTopology(org.apache.storm.generated.StormTopology) Locale(java.util.Locale) Map(java.util.Map) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TimeUtil(org.apache.storm.st.utils.TimeUtil) StormSubmitter(org.apache.storm.StormSubmitter) Logger(org.slf4j.Logger) ExecutorStats(org.apache.storm.generated.ExecutorStats) ImmutableMap(com.google.common.collect.ImmutableMap) MalformedURLException(java.net.MalformedURLException) ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) StringDecorator(org.apache.storm.st.utils.StringDecorator) Collection(java.util.Collection) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) Set(java.util.Set) ExecutorAggregateStats(org.apache.storm.generated.ExecutorAggregateStats) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) TopologyInfo(org.apache.storm.generated.TopologyInfo) FromJson(org.apache.storm.st.topology.window.data.FromJson) Utils(org.apache.storm.utils.Utils) Collectors(java.util.stream.Collectors) TException(org.apache.storm.thrift.TException) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) ExecutorSummary(org.apache.storm.generated.ExecutorSummary) IOUtils(org.apache.commons.io.IOUtils) AuthorizationException(org.apache.storm.generated.AuthorizationException) URLEncoder(java.net.URLEncoder) List(java.util.List) Config(org.apache.storm.Config) Pattern(java.util.regex.Pattern) TopologySummary(org.apache.storm.generated.TopologySummary) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) ExecutorStats(org.apache.storm.generated.ExecutorStats) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) TopologyInfo(org.apache.storm.generated.TopologyInfo) ExecutorSummary(org.apache.storm.generated.ExecutorSummary)

Example 5 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class LocalCluster method submitTopology.

@Override
public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, StormTopology topology) throws TException {
    if (!Utils.isValidConf(conf)) {
        throw new IllegalArgumentException("Topology conf is not json-serializable");
    }
    getNimbus().submitTopology(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology));
    ISubmitterHook hook = (ISubmitterHook) Utils.getConfiguredClass(conf, Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN);
    if (hook != null) {
        TopologyInfo topologyInfo = Utils.getTopologyInfo(topologyName, null, conf);
        try {
            hook.notify(topologyInfo, conf, topology);
        } catch (IllegalAccessException e) {
            throw new RuntimeException(e);
        }
    }
    return new LocalTopology(topologyName, topology);
}
Also used : TopologyInfo(org.apache.storm.generated.TopologyInfo)

Aggregations

TopologyInfo (org.apache.storm.generated.TopologyInfo)23 ExecutorSummary (org.apache.storm.generated.ExecutorSummary)12 GetInfoOptions (org.apache.storm.generated.GetInfoOptions)9 Map (java.util.Map)8 HashMap (java.util.HashMap)7 List (java.util.List)7 ArrayList (java.util.ArrayList)6 TopologySummary (org.apache.storm.generated.TopologySummary)6 ExecutorStats (org.apache.storm.generated.ExecutorStats)5 SpoutStats (org.apache.storm.generated.SpoutStats)5 ImmutableMap (com.google.common.collect.ImmutableMap)4 HashSet (java.util.HashSet)4 AlreadyAliveException (org.apache.storm.generated.AlreadyAliveException)4 AuthorizationException (org.apache.storm.generated.AuthorizationException)4 InvalidTopologyException (org.apache.storm.generated.InvalidTopologyException)4 StormTopology (org.apache.storm.generated.StormTopology)4 File (java.io.File)3 IOException (java.io.IOException)3 Collectors (java.util.stream.Collectors)3 Config (org.apache.storm.Config)3