use of org.apache.storm.generated.ExecutorSummary in project storm by apache.
the class Nimbus method getTopologyInfoWithOpts.
@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
try {
getTopologyInfoWithOptsCalls.mark();
CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
if (common.base == null) {
throw new NotAliveException(topoId);
}
IStormClusterState state = stormClusterState;
NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
Map<String, List<ErrorInfo>> errors = new HashMap<>();
for (String component : common.allComponents) {
switch(numErrChoice) {
case NONE:
errors.put(component, Collections.emptyList());
break;
case ONE:
List<ErrorInfo> errList = new ArrayList<>();
ErrorInfo info = state.lastError(topoId, component);
if (info != null) {
errList.add(info);
}
errors.put(component, errList);
break;
case ALL:
errors.put(component, state.errors(topoId, component));
break;
default:
LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
errors.put(component, state.errors(topoId, component));
break;
}
}
List<ExecutorSummary> summaries = new ArrayList<>();
if (common.assignment != null) {
for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
NodeInfo ni = entry.getValue();
ExecutorInfo execInfo = toExecInfo(entry.getKey());
String host = entry.getValue().get_node();
Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
if (heartbeat == null) {
heartbeat = Collections.emptyMap();
}
ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
//heartbeats "stats"
Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
if (hb != null) {
Map ex = (Map) hb.get("stats");
if (ex != null) {
ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
summ.set_stats(stats);
}
}
summaries.add(summ);
}
}
TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
if (common.base.is_set_owner()) {
topoInfo.set_owner(common.base.get_owner());
}
String schedStatus = idToSchedStatus.get().get(topoId);
if (schedStatus != null) {
topoInfo.set_sched_status(schedStatus);
}
TopologyResources resources = getResourcesForTopology(topoId, common.base);
if (resources != null) {
topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
topoInfo.set_requested_cpu(resources.getRequestedCpu());
topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
topoInfo.set_assigned_cpu(resources.getAssignedCpu());
}
if (common.base.is_set_component_debug()) {
topoInfo.set_component_debug(common.base.get_component_debug());
}
topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
return topoInfo;
} catch (Exception e) {
LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
if (e instanceof TException) {
throw (TException) e;
}
throw new RuntimeException(e);
}
}
use of org.apache.storm.generated.ExecutorSummary in project storm by apache.
the class InOrderDeliveryTest method printMetrics.
public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
TopologyInfo info = client.getTopologyInfoByName(name);
int uptime = info.get_uptime_secs();
long acked = 0;
long failed = 0;
double weightedAvgTotal = 0.0;
for (ExecutorSummary exec : info.get_executors()) {
if ("spout".equals(exec.get_component_id())) {
SpoutStats stats = exec.get_stats().get_specific().get_spout();
Map<String, Long> failedMap = stats.get_failed().get(":all-time");
Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
for (String key : ackedMap.keySet()) {
if (failedMap != null) {
Long tmp = failedMap.get(key);
if (tmp != null) {
failed += tmp;
}
}
long ackVal = ackedMap.get(key);
double latVal = avgLatMap.get(key) * ackVal;
acked += ackVal;
weightedAvgTotal += latVal;
}
}
}
double avgLatency = weightedAvgTotal / acked;
System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " + (((double) acked) / uptime + " failed: " + failed));
}
use of org.apache.storm.generated.ExecutorSummary in project storm by apache.
the class TopoWrap method getComponentExecutorCount.
public long getComponentExecutorCount(final String componentId) throws TException {
TopologyInfo info = getInfo();
List<ExecutorSummary> executors = info.get_executors();
return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).count();
}
use of org.apache.storm.generated.ExecutorSummary in project storm by apache.
the class TopoWrap method getAllTimeEmittedCount.
public long getAllTimeEmittedCount(final String componentId) throws TException {
TopologyInfo info = getInfo();
final List<ExecutorSummary> executors = info.get_executors();
return executors.stream().filter(summary -> summary != null && summary.get_component_id().equals(componentId)).mapToLong(summary -> {
ExecutorStats executorStats = summary.get_stats();
if (executorStats == null) {
return 0L;
}
Map<String, Map<String, Long>> emitted = executorStats.get_emitted();
if (emitted == null) {
return 0L;
}
Map<String, Long> allTime = emitted.get(":all-time");
if (allTime == null) {
return 0L;
}
return allTime.get(Utils.DEFAULT_STREAM_ID);
}).sum();
}
use of org.apache.storm.generated.ExecutorSummary in project storm by apache.
the class TopoWrap method getLogUrls.
/**
* Get the Logviewer worker log URLs for the specified component.
*/
public Set<ExecutorURL> getLogUrls(final String componentId) throws TException, MalformedURLException {
ComponentPageInfo componentPageInfo = cluster.getNimbusClient().getComponentPageInfo(id, componentId, null, false);
List<ExecutorAggregateStats> executorStats = componentPageInfo.get_exec_stats();
Set<ExecutorURL> urls = new HashSet<>();
for (ExecutorAggregateStats execStat : executorStats) {
ExecutorSummary execSummary = execStat.get_exec_summary();
String host = execSummary.get_host();
int executorPort = execSummary.get_port();
// http://supervisor2:8000/download/DemoTest-26-1462229009%2F6703%2Fworker.log
// http://supervisor2:8000/log?file=SlidingWindowCountTest-9-1462388349%2F6703%2Fworker.log
int logViewerPort = 8000;
ExecutorURL executorURL = new ExecutorURL(componentId, host, logViewerPort, executorPort, id);
urls.add(executorURL);
}
return urls;
}
Aggregations