Examples with NimbusClient backtype.storm.utils.NimbusClient used on opensource projects

Example 36 with NimbusClient

use of backtype.storm.utils.NimbusClient in project storm by nathanmarz.

the class StormSubmitter method submitJar.

public static String submitJar(Map conf, String localJar) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String uploadLocation = client.getClient().beginFileUpload();
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        BufferFileInputStream is = new BufferFileInputStream(localJar);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}

Example 37 with NimbusClient

use of backtype.storm.utils.NimbusClient in project storm by nathanmarz.

the class StormSubmitter method submitTopology.

/**
     * Submits a topology to run on the cluster. A topology runs forever or until 
     * explicitly killed.
     *
     *
     * @param name the name of the storm.
     * @param stormConf the topology-specific configuration. See {@link Config}. 
     * @param topology the processing to execute.
     * @param options to manipulate the starting of the topology
     * @throws AlreadyAliveException if a topology with this name is already running
     * @throws InvalidTopologyException if an invalid topology was submitted
     */
public static void submitTopology(String name, Map stormConf, StormTopology topology, SubmitOptions opts) throws AlreadyAliveException, InvalidTopologyException {
    if (!Utils.isValidConf(stormConf)) {
        throw new IllegalArgumentException("Storm conf is not valid. Must be json-serializable");
    }
    stormConf = new HashMap(stormConf);
    stormConf.putAll(Utils.readCommandLineOpts());
    Map conf = Utils.readStormConfig();
    conf.putAll(stormConf);
    try {
        String serConf = JSONValue.toJSONString(stormConf);
        if (localNimbus != null) {
            LOG.info("Submitting topology " + name + " in local mode");
            localNimbus.submitTopology(name, null, serConf, topology);
        } else {
            NimbusClient client = NimbusClient.getConfiguredClient(conf);
            if (topologyNameExists(conf, name)) {
                throw new RuntimeException("Topology with name `" + name + "` already exists on cluster");
            }
            submitJar(conf);
            try {
                LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf);
                if (opts != null) {
                    client.getClient().submitTopologyWithOpts(name, submittedJar, serConf, topology, opts);
                } else {
                    // this is for backwards compatibility
                    client.getClient().submitTopology(name, submittedJar, serConf, topology);
                }
            } catch (InvalidTopologyException e) {
                LOG.warn("Topology submission exception", e);
                throw e;
            } catch (AlreadyAliveException e) {
                LOG.warn("Topology already alive exception", e);
                throw e;
            } finally {
                client.close();
            }
        }
        LOG.info("Finished submitting topology: " + name);
    } catch (TException e) {
        throw new RuntimeException(e);
    }
}

Example 38 with NimbusClient

use of backtype.storm.utils.NimbusClient in project jstorm by alibaba.

the class TopologyAPIController method components.

@RequestMapping("")
public Map components(@PathVariable String clusterName, @PathVariable String topology, @RequestParam(value = "window", required = false) String window) {
    Map<String, Object> ret = new HashMap<>();
    int win = UIUtils.parseWindow(window);
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topology);
        TopologySummary topologySummary = topologyInfo.get_topology();
        // fill in topology summary information
        ret.put("id", topologySummary.get_id());
        ret.put("name", topologySummary.get_name());
        ret.put("uptime", UIUtils.prettyUptime(topologySummary.get_uptimeSecs()));
        ret.put("uptimeSeconds", topologySummary.get_uptimeSecs());
        ret.put("status", topologySummary.get_status());
        ret.put("tasksTotal", topologySummary.get_numTasks());
        ret.put("workersTotal", topologySummary.get_numWorkers());
        ret.put("windowHint", AsmWindow.win2str(win));
        // fill in topology metrics information
        MetricInfo topologyMetrics = topologyInfo.get_metrics().get_topologyMetric();
        UISummaryMetric topologyData = UIMetricUtils.getSummaryMetrics(topologyMetrics, win);
        ret.put("topologyMetrics", topologyData);
        // fill in components metrics information
        MetricInfo componentMetrics = topologyInfo.get_metrics().get_componentMetric();
        List<UIUserDefinedMetric> userDefinedMetrics = new ArrayList<>();
        List<UIComponentMetric> componentData = UIMetricUtils.getComponentMetrics(componentMetrics, win, topologyInfo.get_components(), userDefinedMetrics);
        ret.put("componentMetrics", componentData);
        ret.put("userDefinedMetrics", userDefinedMetrics);
        // fill in workers metrics information
        MetricInfo workerMetrics = topologyInfo.get_metrics().get_workerMetric();
        List<UIWorkerMetric> workerData = UIMetricUtils.getWorkerMetrics(workerMetrics, topology, win);
        ret.put("workerMetrics", workerData);
        // fill in tasks entities information
        List<TaskEntity> taskData = UIUtils.getTaskEntities(topologyInfo);
        ret.put("taskStats", taskData);
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}