Examples with Config backtype.storm.Config used on opensource projects

Example 66 with Config

use of backtype.storm.Config in project storm by nathanmarz.

the class OpaqueMemoryTransactionalSpout method getComponentConfiguration.

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Example 67 with Config

use of backtype.storm.Config in project storm-lib by xumingming.

the class TestingApiDemo method testWithLocalCluster.

public void testWithLocalCluster() {
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setSupervisors(2);
    mkClusterParam.setPortsPerSupervisor(5);
    Config daemonConf = new Config();
    daemonConf.put(Config.SUPERVISOR_ENABLE, false);
    daemonConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, 0);
    /**
		 * when testing your topology, you need a <code>LocalCluster</code> to run your topologies, you need
		 * to create it, after using it, you need to stop it. Using <code>Testing.withLocalCluster</code> you
		 * don't need to do any of this, just use the <code>cluster</code> provided through the param of 
		 * <code>TestJob.run</code>.
		 */
    Testing.withLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            assertNotNull(cluster.getState());
        }
    });
}

Example 68 with Config

use of backtype.storm.Config in project storm-lib by xumingming.

the class TestingApiDemo method testTimeout.

public void testTimeout() {
    Config daemonConfig = new Config();
    daemonConfig.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true);
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setDaemonConf(daemonConfig);
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            AckFailMapTracker tracker = new AckFailMapTracker();
            FeederSpout feeder = createFeederSpout("field1");
            feeder.setAckFailDelegate(tracker);
            TopologyBuilder builder = new TopologyBuilder();
            builder.setSpout("1", feeder);
            builder.setBolt("2", new AckEveryOtherBolt()).globalGrouping("1");
            StormTopology topology = builder.createTopology();
            Config topologyConfig = new Config();
            topologyConfig.setMessageTimeoutSecs(10);
            /**
				 * TODO
				 */
            try {
                cluster.submitTopology("timeout-tester", topologyConfig, topology);
            } catch (AlreadyAliveException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (InvalidTopologyException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            feeder.feed(new Values("a"), 1);
            feeder.feed(new Values("b"), 2);
            feeder.feed(new Values("c"), 3);
            /**
				 * TODO
				 */
            Testing.advanceClusterTime(cluster, 9);
            assertAcked(tracker, 1, 3);
            assertFalse(tracker.isFailed(2));
            Testing.advanceClusterTime(cluster, 12);
            assertFailed(tracker, 2);
        }
    });
}

Example 69 with Config

use of backtype.storm.Config in project pulsar by yahoo.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "com.yahoo.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}

Example 70 with Config

use of backtype.storm.Config in project storm-hbase by ypf412.

the class DumpToHBaseTopology method main.

/**
	 * HBase Data Dump to Another HBase Table Topology
	 * @param args
	 * @throws Exception
	 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    PropConfig pc = new PropConfig("storm.properties");
    int topoWorkers = Integer.valueOf(pc.getProperty("storm.topolopy.workers"));
    int spoutTasks = Integer.valueOf(pc.getProperty("storm.spout.tasks"));
    builder.setSpout("hbaseSpout", new HBaseSpout(), spoutTasks);
    int boltTasks = spoutTasks;
    builder.setBolt("dumpBolt", new DumpToHBaseBolt(), boltTasks).fieldsGrouping("hbaseSpout", new Fields("sharding"));
    Config conf = new Config();
    conf.put(Constants.STORM_PROP_CONF_FILE, "storm.properties");
    conf.put(Constants.HBASE_PROP_CONF_FILE, "hbase.properties");
    if (args != null && args.length > 0) {
        // run on storm cluster
        conf.setNumAckers(1);
        conf.setNumWorkers(topoWorkers);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run on local cluster
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}