Examples with Config backtype.storm.Config used on opensource projects

Example 11 with Config

use of backtype.storm.Config in project storm-hbase by ypf412.

the class OutputTopology method main.

/**
	 * HBase Data Output Topology
	 * @param args
	 * @throws Exception
	 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    PropConfig pc = new PropConfig("storm.properties");
    int topoWorkers = Integer.valueOf(pc.getProperty("storm.topolopy.workers"));
    int spoutTasks = Integer.valueOf(pc.getProperty("storm.spout.tasks"));
    builder.setSpout("hbaseSpout", new HBaseSpout(), spoutTasks);
    int boltTasks = spoutTasks;
    builder.setBolt("outputBolt", new OutputBolt(), boltTasks).fieldsGrouping("hbaseSpout", new Fields("sharding"));
    Config conf = new Config();
    conf.put(Constants.STORM_PROP_CONF_FILE, "storm.properties");
    conf.put(Constants.HBASE_PROP_CONF_FILE, "hbase.properties");
    if (args != null && args.length > 0) {
        // run on storm cluster
        conf.setNumAckers(1);
        conf.setNumWorkers(topoWorkers);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run on local cluster
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}

Example 12 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class JStormUnitTestRunner method submitTopology.

/**
     * submit a topology to run in local cluster mode and check if the result should
     * pass the unit test by a callback.
     * @param topology the topology to submit
     * @param runtimeSec max run time in seconds, minimum is 120s
     * @param validator the callback to invoke after cluster closed
     * @return the result of validator if set, or true if it is null
     */
public static boolean submitTopology(StormTopology topology, Map config, int runtimeSec, JStormUnitTestValidator validator) {
    JStormUtils.sleepMs(15 * 1000);
    if (runtimeSec < 120)
        runtimeSec = 120;
    LocalCluster localCluster = new LocalCluster();
    String topologyName;
    if (config == null)
        config = new Config();
    if (config.containsKey(Config.TOPOLOGY_NAME))
        topologyName = (String) config.get(Config.TOPOLOGY_NAME);
    else
        topologyName = "JStormUnitTestTopology";
    localCluster.submitTopology(topologyName, config, topology);
    JStormUtils.sleepMs(runtimeSec * 1000);
    if (validator != null)
        return validator.validate(config);
    return true;
}

Example 13 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class OpaqueMemoryTransactionalSpout method getComponentConfiguration.

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Example 14 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class TCKTest method testTopologySourceWithConfigMethods.

@Test
public void testTopologySourceWithConfigMethods() throws Exception {
    TopologyDef topologyDef = FluxParser.parseResource("/configs/config-methods-test.yaml", false, true, null, false);
    assertTrue(topologyDef.validate());
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    assertNotNull(topology);
    topology.validate();
    // make sure the property was actually set
    TestBolt bolt = (TestBolt) context.getBolt("bolt-1");
    assertTrue(bolt.getFoo().equals("foo"));
    assertTrue(bolt.getBar().equals("bar"));
    assertTrue(bolt.getFooBar().equals("foobar"));
}

Example 15 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class Flux method runCli.

private static void runCli(CommandLine cmd) throws Exception {
    if (!cmd.hasOption(OPTION_NO_SPLASH)) {
        printSplash();
    }
    boolean dumpYaml = cmd.hasOption("dump-yaml");
    TopologyDef topologyDef = null;
    String filePath = (String) cmd.getArgList().get(0);
    // TODO conditionally load properties from a file our resource
    String filterProps = null;
    if (cmd.hasOption(OPTION_FILTER)) {
        filterProps = cmd.getOptionValue(OPTION_FILTER);
    }
    boolean envFilter = cmd.hasOption(OPTION_ENV_FILTER);
    if (cmd.hasOption(OPTION_RESOURCE)) {
        printf("Parsing classpath resource: %s", filePath);
        topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, filterProps, envFilter);
    } else {
        printf("Parsing file: %s", new File(filePath).getAbsolutePath());
        topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, filterProps, envFilter);
    }
    String topologyName = topologyDef.getName();
    // merge contents of `config` into topology config
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    if (!cmd.hasOption(OPTION_NO_DETAIL)) {
        printTopologyInfo(context);
    }
    if (!cmd.hasOption(OPTION_DRY_RUN)) {
        if (cmd.hasOption(OPTION_REMOTE)) {
            LOG.info("Running remotely...");
            try {
                // should the topology be active or inactive
                SubmitOptions submitOptions = null;
                if (cmd.hasOption(OPTION_INACTIVE)) {
                    LOG.info("Deploying topology in an INACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.INACTIVE);
                } else {
                    LOG.info("Deploying topology in an ACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE);
                }
                StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions);
            } catch (Exception e) {
                LOG.warn("Unable to deploy topology to remote cluster.", e);
            }
        } else {
            LOG.info("Running in local mode...");
            String sleepStr = cmd.getOptionValue(OPTION_SLEEP);
            Long sleepTime = DEFAULT_LOCAL_SLEEP_TIME;
            if (sleepStr != null) {
                sleepTime = Long.parseLong(sleepStr);
            }
            LOG.debug("Sleep time: {}", sleepTime);
            LocalCluster cluster = null;
            // in-process or external zookeeper
            if (cmd.hasOption(OPTION_ZOOKEEPER)) {
                String zkStr = cmd.getOptionValue(OPTION_ZOOKEEPER);
                LOG.info("Using ZooKeeper at '{}' instead of in-process one.", zkStr);
                long zkPort = DEFAULT_ZK_PORT;
                String zkHost = null;
                if (zkStr.contains(":")) {
                    String[] hostPort = zkStr.split(":");
                    zkHost = hostPort[0];
                    zkPort = hostPort.length > 1 ? Long.parseLong(hostPort[1]) : DEFAULT_ZK_PORT;
                } else {
                    zkHost = zkStr;
                }
                // the following constructor is only available in 0.9.3 and later
                /*                    try {
                        cluster = new LocalCluster(zkHost, zkPort);
                    } catch (NoSuchMethodError e){
                        LOG.error("The --zookeeper option can only be used with Apache Storm 0.9.3 and later.");
                        System.exit(1);
                    }*/
                LOG.error("sorry, jstorm don't support this operation!!!");
                System.exit(1);
            } else {
                cluster = new LocalCluster();
            }
            cluster.submitTopology(topologyName, conf, topology);
            Utils.sleep(sleepTime);
            cluster.killTopology(topologyName);
            cluster.shutdown();
        }
    }
}