use of backtype.storm.generated.WorkerSummary in project jstorm by alibaba.
the class ServiceHandler method getSupervisorWorkersByHostOrId.
/**
* gets supervisor workers by host or supervisor id, note that id priors to host.
*
* @param host host
* @param id supervisor id
* @return supervisor workers
*/
private SupervisorWorkers getSupervisorWorkersByHostOrId(String host, String id) throws TException {
long start = System.nanoTime();
if (StringUtils.isBlank(id) && StringUtils.isBlank(host)) {
throw new TException("Must specify host or supervisor id!");
}
try {
StormClusterState stormClusterState = data.getStormClusterState();
// all supervisors
Map<String, SupervisorInfo> supervisorInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
SupervisorInfo supervisorInfo = null;
String ip;
if (!StringUtils.isBlank(id)) {
supervisorInfo = supervisorInfos.get(id);
host = supervisorInfo.getHostName();
ip = NetWorkUtils.host2Ip(host);
} else {
ip = NetWorkUtils.host2Ip(host);
for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
SupervisorInfo info = entry.getValue();
if (info.getHostName().equals(host) || info.getHostName().equals(ip)) {
id = entry.getKey();
supervisorInfo = info;
break;
}
}
}
if (supervisorInfo == null) {
throw new TException("unknown supervisor id:" + id);
}
Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);
Map<Integer, WorkerSummary> portWorkerSummarys = new TreeMap<Integer, WorkerSummary>();
int usedSlotNumber = 0;
Map<String, Map<Integer, String>> topologyTaskToComponent = new HashMap<String, Map<Integer, String>>();
Map<String, MetricInfo> metricInfoMap = new HashMap<String, MetricInfo>();
for (Entry<String, Assignment> entry : assignments.entrySet()) {
String topologyId = entry.getKey();
Assignment assignment = entry.getValue();
Set<ResourceWorkerSlot> workers = assignment.getWorkers();
for (ResourceWorkerSlot worker : workers) {
if (!id.equals(worker.getNodeId())) {
continue;
}
usedSlotNumber++;
Integer port = worker.getPort();
WorkerSummary workerSummary = portWorkerSummarys.get(port);
if (workerSummary == null) {
workerSummary = new WorkerSummary();
workerSummary.set_port(port);
workerSummary.set_topology(topologyId);
workerSummary.set_tasks(new ArrayList<TaskComponent>());
portWorkerSummarys.put(port, workerSummary);
}
Map<Integer, String> taskToComponent = topologyTaskToComponent.get(topologyId);
if (taskToComponent == null) {
taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
topologyTaskToComponent.put(topologyId, taskToComponent);
}
int earliest = TimeUtils.current_time_secs();
for (Integer taskId : worker.getTasks()) {
TaskComponent taskComponent = new TaskComponent();
taskComponent.set_component(taskToComponent.get(taskId));
taskComponent.set_taskId(taskId);
Integer startTime = assignment.getTaskStartTimeSecs().get(taskId);
if (startTime != null && startTime < earliest) {
earliest = startTime;
}
workerSummary.add_to_tasks(taskComponent);
}
workerSummary.set_uptime(TimeUtils.time_delta(earliest));
String workerSlotName = getWorkerSlotName(ip, port);
List<MetricInfo> workerMetricInfoList = this.data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
if (workerMetricInfoList.size() > 0) {
MetricInfo workerMetricInfo = workerMetricInfoList.get(0);
// remove metrics that don't belong to current worker
for (Iterator<String> itr = workerMetricInfo.get_metrics().keySet().iterator(); itr.hasNext(); ) {
String metricName = itr.next();
if (!metricName.contains(ip)) {
itr.remove();
}
}
metricInfoMap.put(workerSlotName, workerMetricInfo);
}
}
}
List<WorkerSummary> workerList = new ArrayList<WorkerSummary>();
workerList.addAll(portWorkerSummarys.values());
Map<String, Integer> supervisorToUsedSlotNum = new HashMap<String, Integer>();
supervisorToUsedSlotNum.put(id, usedSlotNumber);
SupervisorSummary supervisorSummary = NimbusUtils.mkSupervisorSummary(supervisorInfo, id, supervisorToUsedSlotNum);
return new SupervisorWorkers(supervisorSummary, workerList, metricInfoMap);
} catch (TException e) {
LOG.info("Failed to get ClusterSummary ", e);
throw e;
} catch (Exception e) {
LOG.info("Failed to get ClusterSummary ", e);
throw new TException(e);
} finally {
long end = System.nanoTime();
SimpleJStormMetric.updateNimbusHistogram("getSupervisorWorkers", (end - start) / TimeUtils.NS_PER_US);
}
}
use of backtype.storm.generated.WorkerSummary in project jstorm by alibaba.
the class SupervisorController method show.
@RequestMapping(value = "/supervisor", method = RequestMethod.GET)
public String show(@RequestParam(value = "cluster", required = true) String clusterName, @RequestParam(value = "host", required = true) String host, @RequestParam(value = "win", required = false) String win, ModelMap model) {
clusterName = StringEscapeUtils.escapeHtml(clusterName);
long start = System.currentTimeMillis();
host = NetWorkUtils.host2Ip(host);
int window = UIUtils.parseWindow(win);
UIUtils.addWindowAttribute(model, window);
NimbusClient client = null;
try {
client = NimbusClientManager.getNimbusClient(clusterName);
//get supervisor summary
SupervisorWorkers supervisorWorkers = client.getClient().getSupervisorWorkers(host);
model.addAttribute("supervisor", new SupervisorEntity(supervisorWorkers.get_supervisor()));
//get worker summary
List<WorkerSummary> workerSummaries = supervisorWorkers.get_workers();
model.addAttribute("workerSummary", UIUtils.getWorkerEntities(workerSummaries));
//get worker metrics
Map<String, MetricInfo> workerMetricInfo = supervisorWorkers.get_workerMetric();
List<UIWorkerMetric> workerMetrics = UIMetricUtils.getWorkerMetrics(workerMetricInfo, workerSummaries, host, window);
// System.out.println("workerMetricInfo:"+workerMetricInfo);
model.addAttribute("workerMetrics", workerMetrics);
model.addAttribute("workerHead", UIMetricUtils.sortHead(workerMetrics, UIWorkerMetric.HEAD));
} catch (Exception e) {
NimbusClientManager.removeClient(clusterName);
LOG.error(e.getMessage(), e);
UIUtils.addErrorAttribute(model, e);
}
// page information
model.addAttribute("clusterName", clusterName);
model.addAttribute("host", host);
model.addAttribute("page", "supervisor");
model.addAttribute("supervisorPort", UIUtils.getSupervisorPort(clusterName));
UIUtils.addTitleAttribute(model, "Supervisor Summary");
LOG.info("supervisor page show cost:{}ms", System.currentTimeMillis() - start);
return "supervisor";
}
use of backtype.storm.generated.WorkerSummary in project jstorm by alibaba.
the class UIMetricUtils method getWorkerMetrics.
public static List<UIWorkerMetric> getWorkerMetrics(Map<String, MetricInfo> workerMetricInfo, List<WorkerSummary> workerSummaries, String host, int window) {
Map<String, UIWorkerMetric> workerMetrics = new HashMap<>();
for (MetricInfo info : workerMetricInfo.values()) {
if (info != null) {
for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
String name = metric.getKey();
String[] split_name = name.split("@");
String _host = UIMetricUtils.extractComponentName(split_name);
if (!host.equals(_host))
continue;
//only handle the specific host
String port = UIMetricUtils.extractTaskId(split_name);
String key = host + ":" + port;
String metricName = UIMetricUtils.extractMetricName(split_name);
MetricSnapshot snapshot = metric.getValue().get(window);
UIWorkerMetric workerMetric;
if (workerMetrics.containsKey(key)) {
workerMetric = workerMetrics.get(key);
} else {
workerMetric = new UIWorkerMetric(host, port);
workerMetrics.put(key, workerMetric);
}
workerMetric.setMetricValue(snapshot, metricName);
}
}
}
for (WorkerSummary ws : workerSummaries) {
String worker = host + ":" + ws.get_port();
if (workerMetrics.containsKey(worker)) {
workerMetrics.get(worker).setTopology(ws.get_topology());
}
}
return new ArrayList<>(workerMetrics.values());
}
Aggregations