use of backtype.storm.generated.ComponentSummary in project jstorm by alibaba.
the class ServiceHandler method getTopologyInfo.
/**
* Get TopologyInfo, it contain all data of the topology running status
*
* @return TopologyInfo
*/
@Override
public TopologyInfo getTopologyInfo(String topologyId) throws NotAliveException, TException {
long start = System.nanoTime();
StormClusterState stormClusterState = data.getStormClusterState();
try {
// get topology's StormBase
StormBase base = stormClusterState.storm_base(topologyId, null);
if (base == null) {
throw new NotAliveException("No topology of " + topologyId);
}
Assignment assignment = stormClusterState.assignment_info(topologyId, null);
if (assignment == null) {
throw new NotAliveException("No topology of " + topologyId);
}
TopologyTaskHbInfo topologyTaskHbInfo = data.getTasksHeartbeat().get(topologyId);
Map<Integer, TaskHeartbeat> taskHbMap = null;
if (topologyTaskHbInfo != null)
taskHbMap = topologyTaskHbInfo.get_taskHbs();
Map<Integer, TaskInfo> taskInfoMap = Cluster.get_all_taskInfo(stormClusterState, topologyId);
Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, taskInfoMap);
Map<Integer, String> taskToType = Cluster.get_all_task_type(stormClusterState, topologyId, taskInfoMap);
String errorString;
if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
errorString = "Y";
} else {
errorString = "";
}
TopologySummary topologySummary = new TopologySummary();
topologySummary.set_id(topologyId);
topologySummary.set_name(base.getStormName());
topologySummary.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
topologySummary.set_status(base.getStatusString());
topologySummary.set_numTasks(NimbusUtils.getTopologyTaskNum(assignment));
topologySummary.set_numWorkers(assignment.getWorkers().size());
topologySummary.set_errorInfo(errorString);
Map<String, ComponentSummary> componentSummaryMap = new HashMap<String, ComponentSummary>();
HashMap<String, List<Integer>> componentToTasks = JStormUtils.reverse_map(taskToComponent);
for (Entry<String, List<Integer>> entry : componentToTasks.entrySet()) {
String name = entry.getKey();
List<Integer> taskIds = entry.getValue();
if (taskIds == null || taskIds.size() == 0) {
LOG.warn("No task of component " + name);
continue;
}
ComponentSummary componentSummary = new ComponentSummary();
componentSummaryMap.put(name, componentSummary);
componentSummary.set_name(name);
componentSummary.set_type(taskToType.get(taskIds.get(0)));
componentSummary.set_parallel(taskIds.size());
componentSummary.set_taskIds(taskIds);
}
Map<Integer, TaskSummary> taskSummaryMap = new TreeMap<Integer, TaskSummary>();
Map<Integer, List<TaskError>> taskErrors = Cluster.get_all_task_errors(stormClusterState, topologyId);
for (Integer taskId : taskInfoMap.keySet()) {
TaskSummary taskSummary = new TaskSummary();
taskSummaryMap.put(taskId, taskSummary);
taskSummary.set_taskId(taskId);
if (taskHbMap == null) {
taskSummary.set_status("Starting");
taskSummary.set_uptime(0);
} else {
TaskHeartbeat hb = taskHbMap.get(taskId);
if (hb == null) {
taskSummary.set_status("Starting");
taskSummary.set_uptime(0);
} else {
boolean isInactive = NimbusUtils.isTaskDead(data, topologyId, taskId);
if (isInactive)
taskSummary.set_status("INACTIVE");
else
taskSummary.set_status("ACTIVE");
taskSummary.set_uptime(hb.get_uptime());
}
}
if (StringUtils.isBlank(errorString)) {
continue;
}
List<TaskError> taskErrorList = taskErrors.get(taskId);
if (taskErrorList != null && taskErrorList.size() != 0) {
for (TaskError taskError : taskErrorList) {
ErrorInfo errorInfo = new ErrorInfo(taskError.getError(), taskError.getTimSecs(), taskError.getLevel(), taskError.getCode());
taskSummary.add_to_errors(errorInfo);
String component = taskToComponent.get(taskId);
componentSummaryMap.get(component).add_to_errors(errorInfo);
}
}
}
for (ResourceWorkerSlot workerSlot : assignment.getWorkers()) {
String hostname = workerSlot.getHostname();
int port = workerSlot.getPort();
for (Integer taskId : workerSlot.getTasks()) {
TaskSummary taskSummary = taskSummaryMap.get(taskId);
taskSummary.set_host(hostname);
taskSummary.set_port(port);
}
}
TopologyInfo topologyInfo = new TopologyInfo();
topologyInfo.set_topology(topologySummary);
topologyInfo.set_components(JStormUtils.mk_list(componentSummaryMap.values()));
topologyInfo.set_tasks(JStormUtils.mk_list(taskSummaryMap.values()));
// return topology metric & component metric only
List<MetricInfo> tpMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.TOPOLOGY);
List<MetricInfo> compMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.COMPONENT);
List<MetricInfo> workerMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
MetricInfo taskMetric = MetricUtils.mkMetricInfo();
MetricInfo streamMetric = MetricUtils.mkMetricInfo();
MetricInfo nettyMetric = MetricUtils.mkMetricInfo();
MetricInfo tpMetric, compMetric, workerMetric;
if (tpMetricList == null || tpMetricList.size() == 0) {
tpMetric = MetricUtils.mkMetricInfo();
} else {
// get the last min topology metric
tpMetric = tpMetricList.get(tpMetricList.size() - 1);
}
if (compMetricList == null || compMetricList.size() == 0) {
compMetric = MetricUtils.mkMetricInfo();
} else {
compMetric = compMetricList.get(0);
}
if (workerMetricList == null || workerMetricList.size() == 0) {
workerMetric = MetricUtils.mkMetricInfo();
} else {
workerMetric = workerMetricList.get(0);
}
TopologyMetric topologyMetrics = new TopologyMetric(tpMetric, compMetric, workerMetric, taskMetric, streamMetric, nettyMetric);
topologyInfo.set_metrics(topologyMetrics);
return topologyInfo;
} catch (TException e) {
LOG.info("Failed to get topologyInfo " + topologyId, e);
throw e;
} catch (Exception e) {
LOG.info("Failed to get topologyInfo " + topologyId, e);
throw new TException("Failed to get topologyInfo" + topologyId);
} finally {
long end = System.nanoTime();
SimpleJStormMetric.updateNimbusHistogram("getTopologyInfo", (end - start) / TimeUtils.NS_PER_US);
}
}
use of backtype.storm.generated.ComponentSummary in project jstorm by alibaba.
the class UIMetricUtils method getComponentMetrics.
/**
* get all component metrics in the topology
* @param info metric info
* @param window window duration for metrics in seconds
* @param componentSummaries list of component summaries
* @param userDefinedMetrics list of user defined metrics for return
* @return list of component metrics
*/
public static List<UIComponentMetric> getComponentMetrics(MetricInfo info, int window, List<ComponentSummary> componentSummaries, List<UIUserDefinedMetric> userDefinedMetrics) {
Map<String, UIComponentMetric> componentData = new HashMap<>();
if (info != null) {
for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
String name = metric.getKey();
String[] split_name = name.split("@");
String compName = UIMetricUtils.extractComponentName(split_name);
String metricName = UIMetricUtils.extractMetricName(split_name);
String group = UIMetricUtils.extractGroup(split_name);
String parentComp = null;
if (metricName != null && metricName.contains(".")) {
parentComp = metricName.split("\\.")[0];
metricName = metricName.split("\\.")[1];
}
if (!metric.getValue().containsKey(window)) {
LOG.debug("component snapshot {} missing window:{}", metric.getKey(), window);
continue;
}
MetricSnapshot snapshot = metric.getValue().get(window);
if (group != null && group.equals("udf")) {
UIUserDefinedMetric udm = new UIUserDefinedMetric(metricName, compName);
udm.setValue(UIMetricUtils.getMetricValue(snapshot));
udm.setType(snapshot.get_metricType());
userDefinedMetrics.add(udm);
} else {
UIComponentMetric compMetric;
if (componentData.containsKey(compName)) {
compMetric = componentData.get(compName);
} else {
compMetric = new UIComponentMetric(compName);
componentData.put(compName, compMetric);
}
compMetric.setMetricValue(snapshot, parentComp, metricName);
}
}
}
//merge sub metrics
for (UIComponentMetric comp : componentData.values()) {
comp.mergeValue();
}
//combine the summary info into metrics
TreeMap<String, UIComponentMetric> ret = new TreeMap<>();
for (ComponentSummary summary : componentSummaries) {
String compName = summary.get_name();
UIComponentMetric compMetric;
if (componentData.containsKey(compName)) {
compMetric = componentData.get(compName);
compMetric.setParallel(summary.get_parallel());
compMetric.setType(summary.get_type());
compMetric.setErrors(summary.get_errors());
} else {
compMetric = new UIComponentMetric(compName, summary.get_parallel(), summary.get_type());
compMetric.setErrors(summary.get_errors());
componentData.put(compName, compMetric);
}
String key = compMetric.getType() + compName;
if (compName.startsWith("__")) {
key = "a" + key;
}
compMetric.setSortedKey(key);
ret.put(key, compMetric);
}
return new ArrayList<>(ret.descendingMap().values());
}
use of backtype.storm.generated.ComponentSummary in project jstorm by alibaba.
the class UIMetricUtils method getComponentMetric.
/**
* get the specific component metric
* @param info metric info
* @param window window duration for metrics in seconds
* @param compName component name
* @param componentSummaries list of component summaries
* @return the component metric
*/
public static UIComponentMetric getComponentMetric(MetricInfo info, int window, String compName, List<ComponentSummary> componentSummaries) {
UIComponentMetric compMetric = new UIComponentMetric(compName);
if (info != null) {
for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
String name = metric.getKey();
String[] split_name = name.split("@");
String componentName = UIMetricUtils.extractComponentName(split_name);
if (componentName != null && !componentName.equals(compName))
continue;
//only handle the specific component
String metricName = UIMetricUtils.extractMetricName(split_name);
String parentComp = null;
if (metricName != null && metricName.contains(".")) {
parentComp = metricName.split("\\.")[0];
metricName = metricName.split("\\.")[1];
}
MetricSnapshot snapshot = metric.getValue().get(window);
compMetric.setMetricValue(snapshot, parentComp, metricName);
}
}
compMetric.mergeValue();
ComponentSummary summary = null;
for (ComponentSummary cs : componentSummaries) {
if (cs.get_name().equals(compName)) {
summary = cs;
break;
}
}
if (summary != null) {
compMetric.setParallel(summary.get_parallel());
compMetric.setType(summary.get_type());
}
return compMetric;
}
Aggregations