Examples with NimbusClientWrapper backtype.storm.utils.NimbusClientWrapper used on opensource projects

Example 6 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class JStormHelper method killTopology.

public static void killTopology(Map conf, String topologyName) throws Exception {
    NimbusClientWrapper client = new NimbusClientWrapper();
    try {
        client.init(conf);
        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        client.getClient().killTopologyWithOpts(topologyName, killOption);
    } finally {
        client.cleanup();
    }
}

Example 7 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class JStormMetricsReporter method uploadMetricData.

public void uploadMetricData(WorkerUploadMetrics metrics) {
    if (inTopology) {
        //in Worker, we upload data via netty transport
        if (boltOutput != null) {
            LOG.debug("emit metrics through bolt collector.");
            ((BoltCollector) boltOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, null, new Values(JStormServerUtils.getName(host, port), metrics));
        } else if (spoutOutput != null) {
            LOG.debug("emit metrics through spout collector.");
            ((SpoutCollector) spoutOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, new Values(JStormServerUtils.getName(host, port), metrics), null);
        } else {
            LOG.warn("topology:{}, both spout/bolt collectors are null, don't know what to do...", topologyId);
        }
    } else {
        // in supervisor or nimbus, we upload metric data via thrift
        LOG.debug("emit metrics through nimbus client.");
        TopologyMetric tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_workerMetric(metrics.get_allMetrics());
        //UpdateEvent.pushEvent(topologyId, tpMetric);
        try {
            // push metrics via nimbus client
            if (client == null) {
                LOG.warn("nimbus client is null...");
                client = new NimbusClientWrapper();
                client.init(conf);
            }
            client.getClient().uploadTopologyMetrics(topologyId, tpMetric);
        } catch (Throwable ex) {
            LOG.error("upload metrics error:", ex);
            if (client != null) {
                client.cleanup();
                client = null;
            }
        }
    }
//MetricUtils.logMetrics(metrics.get_allMetrics());
}

Example 8 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class TaskHeartbeatUpdater method cleanup.

@Override
public void cleanup() {
    try {
        StormBase stormBase = zkCluster.storm_base(topologyId, null);
        boolean isKilledStatus = stormBase != null && stormBase.getStatus().getStatusType().equals(StatusType.killed);
        if (stormBase == null || isKilledStatus) {
            // wait for pendings to exit
            final long timeoutMilliSeconds = 1000 * ConfigExtension.getTaskCleanupTimeoutSec(stormConf);
            final long start = System.currentTimeMillis();
            while (true) {
                try {
                    long delta = System.currentTimeMillis() - start;
                    if (delta > timeoutMilliSeconds) {
                        LOG.warn("Timeout when waiting others' tasks shutdown");
                        break;
                    }
                    if (checkAllTasksShutdown()) {
                        break;
                    }
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    break;
                }
            }
            LOG.info("notify nimbus that all tasks's status is: {} {} ", spoutsExecutorStatusMap.toString(), boltsExecutorStatusMap.toString());
            if (client == null) {
                client = new NimbusClientWrapper();
                client.init(stormConf);
            }
            client.getClient().notifyThisTopologyTasksIsDead(topologyId);
        }
    } catch (Exception e) {
        if (client != null) {
            client.cleanup();
            client = null;
        }
        LOG.warn("Failed to get topology stormbase from zk", e);
    }
}

Example 9 with NimbusClientWrapper

use of backtype.storm.utils.NimbusClientWrapper in project jstorm by alibaba.

the class MetricsUploader method uploadMetrics.

/**
     * upload metrics sequentially due to thrift frame size limit (15MB)
     */
private void uploadMetrics(TopologyMetric tpMetric) throws Exception {
    long start = System.currentTimeMillis();
    if (tpMetric == null) {
        return;
    }
    try {
        synchronized (lock) {
            if (client == null || !client.isValid()) {
                client = new NimbusClientWrapper();
                client.init(conf);
            }
        }
        MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
        MetricInfo componentMetrics = tpMetric.get_componentMetric();
        MetricInfo taskMetrics = tpMetric.get_taskMetric();
        MetricInfo streamMetrics = tpMetric.get_streamMetric();
        MetricInfo workerMetrics = tpMetric.get_workerMetric();
        MetricInfo nettyMetrics = tpMetric.get_nettyMetric();
        int totalSize = topologyMetrics.get_metrics_size() + componentMetrics.get_metrics_size() + taskMetrics.get_metrics_size() + streamMetrics.get_metrics_size() + workerMetrics.get_metrics_size() + nettyMetrics.get_metrics_size();
        // pressure of nimbus
        if (totalSize < MAX_BATCH_SIZE) {
            client.getClient().uploadTopologyMetrics(topologyId, new TopologyMetric(topologyMetrics, componentMetrics, workerMetrics, taskMetrics, streamMetrics, nettyMetrics));
        } else {
            client.getClient().uploadTopologyMetrics(topologyId, new TopologyMetric(topologyMetrics, componentMetrics, dummy, dummy, dummy, dummy));
            batchUploadMetrics(workerMetrics, MetaType.WORKER);
            batchUploadMetrics(taskMetrics, MetaType.TASK);
            batchUploadMetrics(streamMetrics, MetaType.STREAM);
            batchUploadMetrics(nettyMetrics, MetaType.NETTY);
        }
    } catch (Exception e) {
        String errorInfo = "Failed to upload worker metrics";
        LOG.error("Failed to upload worker metrics ", e);
        if (client != null) {
            client.cleanup();
        }
        zkCluster.report_task_error(context.getTopologyId(), context.getThisTaskId(), errorInfo, ErrorConstants.WARN, ErrorConstants.CODE_USER);
    }
    metricLogger.info("upload metrics, cost:{}", System.currentTimeMillis() - start);
}