use of org.apache.storm.generated.ExecutorStats in project storm by apache.
the class Nimbus method getTopologyInfoWithOpts.
@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
try {
getTopologyInfoWithOptsCalls.mark();
CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
if (common.base == null) {
throw new NotAliveException(topoId);
}
IStormClusterState state = stormClusterState;
NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
Map<String, List<ErrorInfo>> errors = new HashMap<>();
for (String component : common.allComponents) {
switch(numErrChoice) {
case NONE:
errors.put(component, Collections.emptyList());
break;
case ONE:
List<ErrorInfo> errList = new ArrayList<>();
ErrorInfo info = state.lastError(topoId, component);
if (info != null) {
errList.add(info);
}
errors.put(component, errList);
break;
case ALL:
errors.put(component, state.errors(topoId, component));
break;
default:
LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
errors.put(component, state.errors(topoId, component));
break;
}
}
List<ExecutorSummary> summaries = new ArrayList<>();
if (common.assignment != null) {
for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
NodeInfo ni = entry.getValue();
ExecutorInfo execInfo = toExecInfo(entry.getKey());
String host = entry.getValue().get_node();
Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
if (heartbeat == null) {
heartbeat = Collections.emptyMap();
}
ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
//heartbeats "stats"
Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
if (hb != null) {
Map ex = (Map) hb.get("stats");
if (ex != null) {
ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
summ.set_stats(stats);
}
}
summaries.add(summ);
}
}
TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
if (common.base.is_set_owner()) {
topoInfo.set_owner(common.base.get_owner());
}
String schedStatus = idToSchedStatus.get().get(topoId);
if (schedStatus != null) {
topoInfo.set_sched_status(schedStatus);
}
TopologyResources resources = getResourcesForTopology(topoId, common.base);
if (resources != null) {
topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
topoInfo.set_requested_cpu(resources.getRequestedCpu());
topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
topoInfo.set_assigned_cpu(resources.getAssignedCpu());
}
if (common.base.is_set_component_debug()) {
topoInfo.set_component_debug(common.base.get_component_debug());
}
topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
return topoInfo;
} catch (Exception e) {
LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
if (e instanceof TException) {
throw (TException) e;
}
throw new RuntimeException(e);
}
}
use of org.apache.storm.generated.ExecutorStats in project storm by apache.
the class StatsUtil method aggregateSpoutStats.
/**
* aggregate spout stats
*
* @param statsSeq a seq of ExecutorStats
* @param includeSys whether to include system streams
* @return aggregated spout stats: {metric -> win -> global stream id -> value}
*/
public static Map<String, Map> aggregateSpoutStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
// actually Map<String, Map<String, Map<String, Long/Double>>>
Map<String, Map> ret = new HashMap<>();
Map<String, Map<String, Map<String, Long>>> commonStats = aggregateCommonStats(statsSeq);
// filter sys streams if necessary
commonStats = preProcessStreamSummary(commonStats, includeSys);
List<Map<String, Map<String, Long>>> acked = new ArrayList<>();
List<Map<String, Map<String, Long>>> failed = new ArrayList<>();
List<Map<String, Map<String, Double>>> completeLatencies = new ArrayList<>();
for (ExecutorSummary summary : statsSeq) {
ExecutorStats stats = summary.get_stats();
acked.add(stats.get_specific().get_spout().get_acked());
failed.add(stats.get_specific().get_spout().get_failed());
completeLatencies.add(stats.get_specific().get_spout().get_complete_ms_avg());
}
ret.putAll(commonStats);
putKV(ret, ACKED, aggregateCounts(acked));
putKV(ret, FAILED, aggregateCounts(failed));
putKV(ret, COMP_LATENCIES, aggregateAverages(completeLatencies, acked));
return ret;
}
use of org.apache.storm.generated.ExecutorStats in project storm by apache.
the class StatsUtil method convertExecutorZkHbs.
/**
* convert clojure structure to java maps
*/
public static Map<List<Integer>, ExecutorStats> convertExecutorZkHbs(Map executorBeats) {
Map<List<Integer>, ExecutorStats> ret = new HashMap<>();
for (Object executorBeat : executorBeats.entrySet()) {
Map.Entry entry = (Map.Entry) executorBeat;
List startEnd = (List) entry.getKey();
ret.put(convertExecutor(startEnd), (ExecutorStats) entry.getValue());
}
return ret;
}
use of org.apache.storm.generated.ExecutorStats in project storm by apache.
the class StatsUtil method aggregateBoltStats.
/**
* aggregate bolt stats
*
* @param statsSeq a seq of ExecutorStats
* @param includeSys whether to include system streams
* @return aggregated bolt stats: {metric -> win -> global stream id -> value}
*/
public static <T> Map<String, Map> aggregateBoltStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
Map<String, Map> ret = new HashMap<>();
Map<String, Map<String, Map<T, Long>>> commonStats = aggregateCommonStats(statsSeq);
// filter sys streams if necessary
commonStats = preProcessStreamSummary(commonStats, includeSys);
List<Map<String, Map<GlobalStreamId, Long>>> acked = new ArrayList<>();
List<Map<String, Map<GlobalStreamId, Long>>> failed = new ArrayList<>();
List<Map<String, Map<GlobalStreamId, Long>>> executed = new ArrayList<>();
List<Map<String, Map<GlobalStreamId, Double>>> processLatencies = new ArrayList<>();
List<Map<String, Map<GlobalStreamId, Double>>> executeLatencies = new ArrayList<>();
for (ExecutorSummary summary : statsSeq) {
ExecutorStats stat = summary.get_stats();
acked.add(stat.get_specific().get_bolt().get_acked());
failed.add(stat.get_specific().get_bolt().get_failed());
executed.add(stat.get_specific().get_bolt().get_executed());
processLatencies.add(stat.get_specific().get_bolt().get_process_ms_avg());
executeLatencies.add(stat.get_specific().get_bolt().get_execute_ms_avg());
}
mergeMaps(ret, commonStats);
putKV(ret, ACKED, aggregateCounts(acked));
putKV(ret, FAILED, aggregateCounts(failed));
putKV(ret, EXECUTED, aggregateCounts(executed));
putKV(ret, PROC_LATENCIES, aggregateAverages(processLatencies, acked));
putKV(ret, EXEC_LATENCIES, aggregateAverages(executeLatencies, executed));
return ret;
}
use of org.apache.storm.generated.ExecutorStats in project storm by apache.
the class StatsUtil method mkEmptyExecutorZkHbs.
// =====================================================================================
// helper methods
// =====================================================================================
public static Map<List<Integer>, ExecutorStats> mkEmptyExecutorZkHbs(Set executors) {
Map<List<Integer>, ExecutorStats> ret = new HashMap<>();
for (Object executor : executors) {
List startEnd = (List) executor;
ret.put(convertExecutor(startEnd), null);
}
return ret;
}
Aggregations