Examples with NimbusClientWrapper backtype.storm.utils.NimbusClientWrapper used on opensource projects

Example 1 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class JStormUtils method getMetrics.

/**
     * Get Topology Metrics
     *
     * @param topologyName topology name
     * @param metricType,  please refer to MetaType, default to MetaType.TASK.getT()
     * @param window,      please refer to AsmWindow, default to AsmWindow.M1_WINDOW
     * @return
     */
public static Map<String, Double> getMetrics(Map conf, String topologyName, MetaType metricType, Integer window) {
    NimbusClientWrapper nimbusClient = null;
    Iface client = null;
    Map<String, Double> summary = new HashMap<>();
    try {
        nimbusClient = new NimbusClientWrapper();
        nimbusClient.init(conf);
        client = nimbusClient.getClient();
        String topologyId = client.getTopologyId(topologyName);
        if (metricType == null) {
            metricType = MetaType.TASK;
        }
        List<MetricInfo> allTaskMetrics = client.getMetrics(topologyId, metricType.getT());
        if (allTaskMetrics == null) {
            throw new RuntimeException("Failed to get metrics");
        }
        if (window == null || !AsmWindow.TIME_WINDOWS.contains(window)) {
            window = AsmWindow.M1_WINDOW;
        }
        for (MetricInfo taskMetrics : allTaskMetrics) {
            Map<String, Map<Integer, MetricSnapshot>> metrics = taskMetrics.get_metrics();
            if (metrics == null) {
                System.out.println("Failed to get task metrics");
                continue;
            }
            for (Entry<String, Map<Integer, MetricSnapshot>> entry : metrics.entrySet()) {
                String key = entry.getKey();
                MetricSnapshot metric = entry.getValue().get(window);
                if (metric == null) {
                    throw new RuntimeException("Failed to get one minute metrics of " + key);
                }
                if (metric.get_metricType() == MetricType.COUNTER.getT()) {
                    summary.put(key, (double) metric.get_longValue());
                } else if (metric.get_metricType() == MetricType.GAUGE.getT()) {
                    summary.put(key, metric.get_doubleValue());
                } else {
                    summary.put(key, metric.get_mean());
                }
            }
        }
        return summary;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            nimbusClient.cleanup();
        }
    }
}

Example 2 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class MetricsRegister method registerMetrics.

public Map<String, Long> registerMetrics(Set<String> names) {
    if (!JStormMetrics.enabled) {
        return new HashMap<>();
    }
    try {
        synchronized (lock) {
            if (client == null) {
                client = new NimbusClientWrapper();
                client.init(conf);
            }
        }
        return client.getClient().registerMetrics(topologyId, names);
    } catch (Exception e) {
        LOG.error("Failed to gen metric ids", e);
        if (client != null) {
            client.cleanup();
            client = null;
        }
    }
    return new HashMap<>();
}

Example 3 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class SupervisorRefreshConfig method run.

@Override
public void run() {
    try {
        if (this.nimbusClientWrapper == null) {
            this.nimbusClientWrapper = new NimbusClientWrapper();
            try {
                this.nimbusClientWrapper.init(this.stormConf);
            } catch (Exception ex) {
                LOG.error("init nimbus client wrapper error, maybe nimbus is not alive.");
            }
        }
        String nimbusYaml = JStormUtils.trimEnd(yarnConfigBlacklist.filterConfigIfNecessary(this.nimbusClientWrapper.getClient().getStormRawConf()));
        Map nimbusConf = LoadConf.loadYamlFromString(nimbusYaml);
        if (nimbusYaml != null && !this.stormConf.equals(nimbusConf)) {
            Map newConf = LoadConf.loadYamlFromString(nimbusYaml);
            if (newConf == null) {
                LOG.error("received invalid storm.yaml, skip...");
            } else {
                LOG.debug("received nimbus config update, new config:\n{}", nimbusYaml);
                this.stormYaml = nimbusYaml;
                // append yarn config
                nimbusYaml = JStormUtils.trimEnd(nimbusYaml + "\n" + retainedYarnConfig);
                // backup config & overwrite current storm.yaml
                RefreshableComponents.refresh(newConf);
                LoadConf.backupAndOverwriteStormYaml(nimbusYaml);
            }
        }
    } catch (Exception ex) {
        LOG.error("failed to get nimbus conf, maybe nimbus is not alive.");
        this.nimbusClientWrapper.reconnect();
    }
}

Example 4 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class TaskHeartbeatUpdater method process.

@Override
public void process(Object event) throws Exception {
    synchronized (_lock) {
        if (event instanceof UpdateConfigEvent) {
            update(((UpdateConfigEvent) event).getConf());
            return;
        }
        Tuple input = (Tuple) event;
        int sourceTask = input.getSourceTask();
        int uptime = (Integer) input.getValue(0);
        TaskStatus executorStatus = new TaskStatus();
        if (input.getValues().size() < 2) {
            // for compatibility
            executorStatus.setStatus(TaskStatus.RUN);
        } else {
            executorStatus.setStatus((byte) input.getValue(1));
        }
        boolean isSendSpoutTaskFinishStream = false;
        if (spoutsExecutorStatusMap.containsKey(sourceTask)) {
            spoutsExecutorStatusMap.put(sourceTask, executorStatus);
        } else if (boltsExecutorStatusMap.containsKey(sourceTask)) {
            boltsExecutorStatusMap.put(sourceTask, executorStatus);
        } else if (sourceTask != taskId) {
            LOG.warn("received invalid task heartbeat {}", input);
        }
        if (executorStatus.getStatus() == TaskStatus.INIT && spoutsExecutorStatusMap.get(sourceTask) != null) {
            boolean existInitStatusBolt = false;
            for (TaskStatus status : boltsExecutorStatusMap.values()) {
                if (status.getStatus() == TaskStatus.INIT || status.getStatus() == TaskStatus.SHUTDOWN) {
                    existInitStatusBolt = true;
                    break;
                }
            }
            if (!existInitStatusBolt)
                isSendSpoutTaskFinishStream = true;
        }
        if (client == null) {
            client = new NimbusClientWrapper();
            client.init(stormConf);
        }
        // Update the heartbeat for source task, but don't update it if task is initial status
        if (executorStatus.getStatus() != TaskStatus.INIT && uptime > 0) {
            TaskHeartbeat taskHb = taskHbMap.get().get(sourceTask);
            if (taskHb == null) {
                taskHb = new TaskHeartbeat(TimeUtils.current_time_secs(), uptime);
                TaskHeartbeat tmpTaskHb = taskHbMap.get().putIfAbsent(sourceTask, taskHb);
                if (tmpTaskHb != null) {
                    taskHb = tmpTaskHb;
                }
            }
            taskHb.set_time(TimeUtils.current_time_secs());
            taskHb.set_uptime(uptime);
        } else if (isSendSpoutTaskFinishStream) {
            TopoMasterCtrlEvent finishInitEvent = new TopoMasterCtrlEvent(TopoMasterCtrlEvent.EventType.topologyFinishInit);
            ((BoltCollector) (collector.getDelegate())).emitDirectCtrl(sourceTask, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(finishInitEvent));
            LOG.info("all bolts' task finish init operation, so tm will notify the spout task-{}", sourceTask);
        }
        // Send heartbeat info of all tasks to nimbus
        if (sourceTask == taskId) {
            uploadHB();
        }
    }
}

Example 5 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class JStormHelper method checkError.

public static void checkError(Map conf, String topologyName) throws Exception {
    NimbusClientWrapper client = new NimbusClientWrapper();
    try {
        client.init(conf);
        String topologyId = client.getClient().getTopologyId(topologyName);
        Map<Integer, List<TaskError>> errors = getTaskErrors(topologyId, conf);
        for (Entry<Integer, List<TaskError>> entry : errors.entrySet()) {
            Integer taskId = entry.getKey();
            List<TaskError> errorList = entry.getValue();
            for (TaskError error : errorList) {
                if (ErrorConstants.ERROR.equals(error.getLevel())) {
                    Assert.fail(taskId + " occur error:" + error.getError());
                } else if (ErrorConstants.FATAL.equals(error.getLevel())) {
                    Assert.fail(taskId + " occur error:" + error.getError());
                }
            }
        }
    } finally {
        client.cleanup();
    }
}