use of backtype.storm.generated.TopologySummary in project jstorm by alibaba.
the class Monitor method getComponents.
/**
* @@@ Don't be compatible with Storm
*
* Here skip the logic
* @param client
* @param topology
* @return
* @throws Exception
*/
private HashSet<String> getComponents(Nimbus.Client client, String topology) throws Exception {
HashSet<String> components = new HashSet<String>();
ClusterSummary clusterSummary = client.getClusterInfo();
TopologySummary topologySummary = null;
for (TopologySummary ts : clusterSummary.get_topologies()) {
if (topology.equals(ts.get_name())) {
topologySummary = ts;
break;
}
}
if (topologySummary == null) {
throw new IllegalArgumentException("topology: " + topology + " not found");
} else {
String id = topologySummary.get_id();
// GetInfoOptions getInfoOpts = new GetInfoOptions();
// getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
// TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
// for (ExecutorSummary es: info.get_executors()) {
// components.add(es.get_component_id());
// }
}
return components;
}
use of backtype.storm.generated.TopologySummary in project jstorm by alibaba.
the class Monitor method metrics.
public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
long totalStatted = 0;
int componentParallelism = 0;
boolean streamFound = false;
ClusterSummary clusterSummary = client.getClusterInfo();
TopologySummary topologySummary = null;
for (TopologySummary ts : clusterSummary.get_topologies()) {
if (_topology.equals(ts.get_name())) {
topologySummary = ts;
break;
}
}
if (topologySummary == null) {
throw new IllegalArgumentException("topology: " + _topology + " not found");
} else {
// String id = topologySummary.get_id();
// GetInfoOptions getInfoOpts = new GetInfoOptions();
// getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
// TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
// for (ExecutorSummary es: info.get_executors()) {
// if (_component.equals(es.get_component_id())) {
// componentParallelism ++;
// ExecutorStats stats = es.get_stats();
// if (stats != null) {
// Map<String,Map<String,Long>> statted =
// WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
// if ( statted != null) {
// Map<String, Long> e2 = statted.get(":all-time");
// if (e2 != null) {
// Long stream = e2.get(_stream);
// if (stream != null){
// streamFound = true;
// totalStatted += stream;
// }
// }
// }
// }
// }
// }
}
if (componentParallelism <= 0) {
HashSet<String> components = getComponents(client, _topology);
System.out.println("Available components for " + _topology + " :");
System.out.println("------------------");
for (String comp : components) {
System.out.println(comp);
}
System.out.println("------------------");
throw new IllegalArgumentException("component: " + _component + " not found");
}
if (!streamFound) {
throw new IllegalArgumentException("stream: " + _stream + " not found");
}
long timeDelta = now - state.getLastTime();
long stattedDelta = totalStatted - state.getLastStatted();
state.setLastTime(now);
state.setLastStatted(totalStatted);
double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
System.out.println(_topology + "\t" + _component + "\t" + componentParallelism + "\t" + _stream + "\t" + timeDelta + "\t" + stattedDelta + "\t" + throughput);
}
use of backtype.storm.generated.TopologySummary in project jstorm by alibaba.
the class ClusterInfoBolt method getClusterInfo.
private void getClusterInfo(Client client) {
try {
ClusterSummary clusterSummary = client.getClusterInfo();
List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
int totalWorkers = 0;
int usedWorkers = 0;
for (SupervisorSummary summary : supervisorSummaryList) {
totalWorkers += summary.get_num_workers();
usedWorkers += summary.get_num_used_workers();
}
int freeWorkers = totalWorkers - usedWorkers;
LOGGER.info("cluster totalWorkers = " + totalWorkers + ", usedWorkers = " + usedWorkers + ", freeWorkers = " + freeWorkers);
HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
long clusterTPS = 0l;
for (TopologySummary topology : topologySummaryList) {
long topologyTPS = getTopologyTPS(topology, client);
clusterTPS += topologyTPS;
if (topology.get_name().startsWith("ClusterMonitor")) {
continue;
}
HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
}
HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
} catch (TException e) {
initClient(configMap);
LOGGER.error("get client info error.", e);
} catch (NotAliveException nae) {
LOGGER.warn("topology is dead.", nae);
}
}
use of backtype.storm.generated.TopologySummary in project jstorm by alibaba.
the class ServiceHandler method getTopologyInfo.
/**
* Get TopologyInfo, it contain all data of the topology running status
*
* @return TopologyInfo
*/
@Override
public TopologyInfo getTopologyInfo(String topologyId) throws NotAliveException, TException {
long start = System.nanoTime();
StormClusterState stormClusterState = data.getStormClusterState();
try {
// get topology's StormBase
StormBase base = stormClusterState.storm_base(topologyId, null);
if (base == null) {
throw new NotAliveException("No topology of " + topologyId);
}
Assignment assignment = stormClusterState.assignment_info(topologyId, null);
if (assignment == null) {
throw new NotAliveException("No topology of " + topologyId);
}
TopologyTaskHbInfo topologyTaskHbInfo = data.getTasksHeartbeat().get(topologyId);
Map<Integer, TaskHeartbeat> taskHbMap = null;
if (topologyTaskHbInfo != null)
taskHbMap = topologyTaskHbInfo.get_taskHbs();
Map<Integer, TaskInfo> taskInfoMap = Cluster.get_all_taskInfo(stormClusterState, topologyId);
Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, taskInfoMap);
Map<Integer, String> taskToType = Cluster.get_all_task_type(stormClusterState, topologyId, taskInfoMap);
String errorString;
if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
errorString = "Y";
} else {
errorString = "";
}
TopologySummary topologySummary = new TopologySummary();
topologySummary.set_id(topologyId);
topologySummary.set_name(base.getStormName());
topologySummary.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
topologySummary.set_status(base.getStatusString());
topologySummary.set_numTasks(NimbusUtils.getTopologyTaskNum(assignment));
topologySummary.set_numWorkers(assignment.getWorkers().size());
topologySummary.set_errorInfo(errorString);
Map<String, ComponentSummary> componentSummaryMap = new HashMap<String, ComponentSummary>();
HashMap<String, List<Integer>> componentToTasks = JStormUtils.reverse_map(taskToComponent);
for (Entry<String, List<Integer>> entry : componentToTasks.entrySet()) {
String name = entry.getKey();
List<Integer> taskIds = entry.getValue();
if (taskIds == null || taskIds.size() == 0) {
LOG.warn("No task of component " + name);
continue;
}
ComponentSummary componentSummary = new ComponentSummary();
componentSummaryMap.put(name, componentSummary);
componentSummary.set_name(name);
componentSummary.set_type(taskToType.get(taskIds.get(0)));
componentSummary.set_parallel(taskIds.size());
componentSummary.set_taskIds(taskIds);
}
Map<Integer, TaskSummary> taskSummaryMap = new TreeMap<Integer, TaskSummary>();
Map<Integer, List<TaskError>> taskErrors = Cluster.get_all_task_errors(stormClusterState, topologyId);
for (Integer taskId : taskInfoMap.keySet()) {
TaskSummary taskSummary = new TaskSummary();
taskSummaryMap.put(taskId, taskSummary);
taskSummary.set_taskId(taskId);
if (taskHbMap == null) {
taskSummary.set_status("Starting");
taskSummary.set_uptime(0);
} else {
TaskHeartbeat hb = taskHbMap.get(taskId);
if (hb == null) {
taskSummary.set_status("Starting");
taskSummary.set_uptime(0);
} else {
boolean isInactive = NimbusUtils.isTaskDead(data, topologyId, taskId);
if (isInactive)
taskSummary.set_status("INACTIVE");
else
taskSummary.set_status("ACTIVE");
taskSummary.set_uptime(hb.get_uptime());
}
}
if (StringUtils.isBlank(errorString)) {
continue;
}
List<TaskError> taskErrorList = taskErrors.get(taskId);
if (taskErrorList != null && taskErrorList.size() != 0) {
for (TaskError taskError : taskErrorList) {
ErrorInfo errorInfo = new ErrorInfo(taskError.getError(), taskError.getTimSecs(), taskError.getLevel(), taskError.getCode());
taskSummary.add_to_errors(errorInfo);
String component = taskToComponent.get(taskId);
componentSummaryMap.get(component).add_to_errors(errorInfo);
}
}
}
for (ResourceWorkerSlot workerSlot : assignment.getWorkers()) {
String hostname = workerSlot.getHostname();
int port = workerSlot.getPort();
for (Integer taskId : workerSlot.getTasks()) {
TaskSummary taskSummary = taskSummaryMap.get(taskId);
taskSummary.set_host(hostname);
taskSummary.set_port(port);
}
}
TopologyInfo topologyInfo = new TopologyInfo();
topologyInfo.set_topology(topologySummary);
topologyInfo.set_components(JStormUtils.mk_list(componentSummaryMap.values()));
topologyInfo.set_tasks(JStormUtils.mk_list(taskSummaryMap.values()));
// return topology metric & component metric only
List<MetricInfo> tpMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.TOPOLOGY);
List<MetricInfo> compMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.COMPONENT);
List<MetricInfo> workerMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
MetricInfo taskMetric = MetricUtils.mkMetricInfo();
MetricInfo streamMetric = MetricUtils.mkMetricInfo();
MetricInfo nettyMetric = MetricUtils.mkMetricInfo();
MetricInfo tpMetric, compMetric, workerMetric;
if (tpMetricList == null || tpMetricList.size() == 0) {
tpMetric = MetricUtils.mkMetricInfo();
} else {
// get the last min topology metric
tpMetric = tpMetricList.get(tpMetricList.size() - 1);
}
if (compMetricList == null || compMetricList.size() == 0) {
compMetric = MetricUtils.mkMetricInfo();
} else {
compMetric = compMetricList.get(0);
}
if (workerMetricList == null || workerMetricList.size() == 0) {
workerMetric = MetricUtils.mkMetricInfo();
} else {
workerMetric = workerMetricList.get(0);
}
TopologyMetric topologyMetrics = new TopologyMetric(tpMetric, compMetric, workerMetric, taskMetric, streamMetric, nettyMetric);
topologyInfo.set_metrics(topologyMetrics);
return topologyInfo;
} catch (TException e) {
LOG.info("Failed to get topologyInfo " + topologyId, e);
throw e;
} catch (Exception e) {
LOG.info("Failed to get topologyInfo " + topologyId, e);
throw new TException("Failed to get topologyInfo" + topologyId);
} finally {
long end = System.nanoTime();
SimpleJStormMetric.updateNimbusHistogram("getTopologyInfo", (end - start) / TimeUtils.NS_PER_US);
}
}
use of backtype.storm.generated.TopologySummary in project jstorm by alibaba.
the class NimbusUtils method getTopologySummary.
public static List<TopologySummary> getTopologySummary(StormClusterState stormClusterState, Map<String, Assignment> assignments) throws Exception {
List<TopologySummary> topologySummaries = new ArrayList<TopologySummary>();
// get all active topology's StormBase
Map<String, StormBase> bases = Cluster.get_all_StormBase(stormClusterState);
for (Entry<String, StormBase> entry : bases.entrySet()) {
String topologyId = entry.getKey();
StormBase base = entry.getValue();
Assignment assignment = stormClusterState.assignment_info(topologyId, null);
if (assignment == null) {
LOG.error("Failed to get assignment of " + topologyId);
continue;
}
assignments.put(topologyId, assignment);
int num_workers = assignment.getWorkers().size();
int num_tasks = getTopologyTaskNum(assignment);
String errorString = null;
if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
errorString = "Y";
} else {
errorString = "";
}
TopologySummary topology = new TopologySummary();
topology.set_id(topologyId);
topology.set_name(base.getStormName());
topology.set_status(base.getStatusString());
topology.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
topology.set_numWorkers(num_workers);
topology.set_numTasks(num_tasks);
topology.set_errorInfo(errorString);
topologySummaries.add(topology);
}
return topologySummaries;
}
Aggregations