Examples with LocalCluster backtype.storm.LocalCluster used on opensource projects

Example 11 with LocalCluster

use of backtype.storm.LocalCluster in project storm-hbase by jrkinley.

the class HBaseCountersBatchTopology method main.

/**
 * @param args
 * @throws InterruptedException
 */
public static void main(String[] args) throws InterruptedException {
    // Add transactional spout
    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(values, new Fields("shortid", "url", "user", "date"), 3);
    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("shorturl-count", "spout", spout, 2);
    // Build TupleTableConifg
    TupleTableConfig ttConfig = new TupleTableConfig("shorturl", "shortid");
    ttConfig.setBatch(false);
    ttConfig.addColumn("data", "clicks");
    ttConfig.addColumn("daily", "date");
    builder.setBolt("hbase-counters", new HBaseCountersBatchBolt(ttConfig), 2).fieldsGrouping("spout", new Fields("shortid"));
    LocalCluster cluster = new LocalCluster();
    Config stormConfig = new Config();
    stormConfig.setDebug(true);
    stormConfig.setMaxSpoutPending(3);
    cluster.submitTopology("hbase-example", stormConfig, builder.buildTopology());
    Thread.sleep(10000);
    cluster.shutdown();
}

Example 12 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class JStormHelper method runTopologyLocally.

public static void runTopologyLocally(StormTopology topology, String topologyName, Config conf, int runtimeInSeconds, Callback callback) throws Exception {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(topologyName, conf, topology);
    if (runtimeInSeconds < 120) {
        JStormUtils.sleepMs(120 * 1000);
    } else {
        JStormUtils.sleepMs(runtimeInSeconds * 1000);
    }
    if (callback != null) {
        callback.execute(topologyName);
    }
    cluster.killTopology(topologyName);
    cluster.shutdown();
}

Example 13 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class JStormUnitTestRunner method submitTopology.

/**
 * submit a topology to run in local cluster mode and check if the result should
 * pass the unit test by a callback.
 * @param topology the topology to submit
 * @param runtimeSec max run time in seconds, minimum is 120s
 * @param validator the callback to invoke after cluster closed
 * @return the result of validator if set, or true if it is null
 */
public static boolean submitTopology(StormTopology topology, Map config, int runtimeSec, JStormUnitTestValidator validator) {
    JStormUtils.sleepMs(15 * 1000);
    if (runtimeSec < 120)
        runtimeSec = 120;
    LocalCluster localCluster = new LocalCluster();
    String topologyName;
    if (config == null)
        config = new Config();
    if (config.containsKey(Config.TOPOLOGY_NAME))
        topologyName = (String) config.get(Config.TOPOLOGY_NAME);
    else
        topologyName = "JStormUnitTestTopology";
    localCluster.submitTopology(topologyName, config, topology);
    JStormUtils.sleepMs(runtimeSec * 1000);
    if (validator != null)
        return validator.validate(config);
    return true;
}

Example 14 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class Flux method runCli.

private static void runCli(CommandLine cmd) throws Exception {
    if (!cmd.hasOption(OPTION_NO_SPLASH)) {
        printSplash();
    }
    boolean dumpYaml = cmd.hasOption("dump-yaml");
    TopologyDef topologyDef = null;
    String filePath = (String) cmd.getArgList().get(0);
    // TODO conditionally load properties from a file our resource
    String filterProps = null;
    if (cmd.hasOption(OPTION_FILTER)) {
        filterProps = cmd.getOptionValue(OPTION_FILTER);
    }
    boolean envFilter = cmd.hasOption(OPTION_ENV_FILTER);
    if (cmd.hasOption(OPTION_RESOURCE)) {
        printf("Parsing classpath resource: %s", filePath);
        topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, filterProps, envFilter);
    } else {
        printf("Parsing file: %s", new File(filePath).getAbsolutePath());
        topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, filterProps, envFilter);
    }
    String topologyName = topologyDef.getName();
    // merge contents of `config` into topology config
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    if (!cmd.hasOption(OPTION_NO_DETAIL)) {
        printTopologyInfo(context);
    }
    if (!cmd.hasOption(OPTION_DRY_RUN)) {
        if (cmd.hasOption(OPTION_REMOTE)) {
            LOG.info("Running remotely...");
            try {
                // should the topology be active or inactive
                SubmitOptions submitOptions = null;
                if (cmd.hasOption(OPTION_INACTIVE)) {
                    LOG.info("Deploying topology in an INACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.INACTIVE);
                } else {
                    LOG.info("Deploying topology in an ACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE);
                }
                StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions);
            } catch (Exception e) {
                LOG.warn("Unable to deploy topology to remote cluster.", e);
            }
        } else {
            LOG.info("Running in local mode...");
            String sleepStr = cmd.getOptionValue(OPTION_SLEEP);
            Long sleepTime = DEFAULT_LOCAL_SLEEP_TIME;
            if (sleepStr != null) {
                sleepTime = Long.parseLong(sleepStr);
            }
            LOG.debug("Sleep time: {}", sleepTime);
            LocalCluster cluster = null;
            // in-process or external zookeeper
            if (cmd.hasOption(OPTION_ZOOKEEPER)) {
                String zkStr = cmd.getOptionValue(OPTION_ZOOKEEPER);
                LOG.info("Using ZooKeeper at '{}' instead of in-process one.", zkStr);
                long zkPort = DEFAULT_ZK_PORT;
                String zkHost = null;
                if (zkStr.contains(":")) {
                    String[] hostPort = zkStr.split(":");
                    zkHost = hostPort[0];
                    zkPort = hostPort.length > 1 ? Long.parseLong(hostPort[1]) : DEFAULT_ZK_PORT;
                } else {
                    zkHost = zkStr;
                }
                // the following constructor is only available in 0.9.3 and later
                /*                    try {
                        cluster = new LocalCluster(zkHost, zkPort);
                    } catch (NoSuchMethodError e){
                        LOG.error("The --zookeeper option can only be used with Apache Storm 0.9.3 and later.");
                        System.exit(1);
                    }*/
                LOG.error("sorry, jstorm don't support this operation!!!");
                System.exit(1);
            } else {
                cluster = new LocalCluster();
            }
            cluster.submitTopology(topologyName, conf, topology);
            Utils.sleep(sleepTime);
            cluster.killTopology(topologyName);
            cluster.shutdown();
        }
    }
}

Example 15 with LocalCluster

use of backtype.storm.LocalCluster in project heron by twitter.

the class ExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    int parallelism = 2;
    builder.setSpout("word", new TestWordSpout(), parallelism);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2 * parallelism).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam("word", ByteAmount.fromGigabytes(3));
    conf.setComponentRam("exclaim1", ByteAmount.fromGigabytes(3));
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(5));
    conf.setContainerCpuRequested(5);
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(parallelism);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        System.out.println("Topology name not provided as an argument, running in simulator mode.");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}