Search in sources :

Example 21 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class TridentMapExample method main.

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    if (args.length == 0) {
        try (LocalDRPC drpc = new LocalDRPC();
            LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(drpc))) {
            for (int i = 0; i < 100; i++) {
                System.out.println("DRPC RESULT: " + drpc.execute("words", "CAT THE DOG JUMPED"));
                Thread.sleep(1000);
            }
        }
    } else {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) Config(org.apache.storm.Config) LocalDRPC(org.apache.storm.LocalDRPC) LocalTopology(org.apache.storm.LocalCluster.LocalTopology)

Example 22 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class TridentMinMaxOfDevicesTopology method main.

public static void main(String[] args) throws Exception {
    StormTopology topology = buildDevicesTopology();
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    if (args.length == 0) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("devices-topology", conf, topology)) {
            Utils.sleep(60 * 1000);
        }
        System.exit(0);
    } else {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar("devices-topology", conf, topology);
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) LocalTopology(org.apache.storm.LocalCluster.LocalTopology)

Example 23 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class TestPlanCompiler method runTridentTopology.

private void runTridentTopology(final int expectedValueSize, AbstractTridentProcessor proc, TridentTopology topo) throws Exception {
    final Config conf = new Config();
    conf.setMaxSpoutPending(20);
    if (proc.getClassLoaders() != null && proc.getClassLoaders().size() > 0) {
        CompilingClassLoader lastClassloader = proc.getClassLoaders().get(proc.getClassLoaders().size() - 1);
        Utils.setClassLoaderForJavaDeSerialize(lastClassloader);
    }
    try (LocalTopology stormTopo = cluster.submitTopology("storm-sql", conf, topo.build())) {
        waitForCompletion(1000 * 1000, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return getCollectedValues().size() < expectedValueSize;
            }
        });
    } finally {
        while (cluster.getClusterInfo().get_topologies_size() > 0) {
            Thread.sleep(10);
        }
        Utils.resetClassLoaderForJavaDeSerialize();
    }
}
Also used : CompilingClassLoader(org.apache.storm.sql.javac.CompilingClassLoader) Config(org.apache.storm.Config) LocalTopology(org.apache.storm.LocalCluster.LocalTopology)

Example 24 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class ConfigurableTopology method submit.

/** Submits the topology under a specific name **/
protected int submit(String name, Config conf, TopologyBuilder builder) {
    if (isLocal) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(name, conf, builder.createTopology())) {
            if (ttl != -1) {
                Utils.sleep(ttl * 1000);
                cluster.shutdown();
            }
        } catch (Exception e) {
            e.printStackTrace();
            return -1;
        }
    } else {
        try {
            StormSubmitter.submitTopology(name, conf, builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
            return -1;
        }
    }
    return 0;
}
Also used : LocalCluster(org.apache.storm.LocalCluster) LocalTopology(org.apache.storm.LocalCluster.LocalTopology) FileNotFoundException(java.io.FileNotFoundException)

Example 25 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class HdfsFileTopology method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);
    SentenceSpout spout = new SentenceSpout();
    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/foo/").withExtension(".txt");
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter("|");
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);
    HdfsBolt bolt = new HdfsBolt().withConfigKey("hdfs.config").withFsUrl(args[0]).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy).addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4).shuffleGrouping(SENTENCE_SPOUT_ID);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology())) {
            waitForSeconds(120);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) RecordFormat(org.apache.storm.hdfs.bolt.format.RecordFormat) DelimitedRecordFormat(org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) TimedRotationPolicy(org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) Yaml(org.yaml.snakeyaml.Yaml) FileInputStream(java.io.FileInputStream) LocalTopology(org.apache.storm.LocalCluster.LocalTopology) MoveFileAction(org.apache.storm.hdfs.common.rotation.MoveFileAction) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

LocalTopology (org.apache.storm.LocalCluster.LocalTopology)54 LocalCluster (org.apache.storm.LocalCluster)52 Config (org.apache.storm.Config)50 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)24 Fields (org.apache.storm.tuple.Fields)17 Map (java.util.Map)7 HashMap (java.util.HashMap)6 JedisPoolConfig (org.apache.storm.redis.common.config.JedisPoolConfig)5 FileInputStream (java.io.FileInputStream)4 InputStream (java.io.InputStream)4 LocalDRPC (org.apache.storm.LocalDRPC)4 StormTopology (org.apache.storm.generated.StormTopology)4 RandomIntegerSpout (org.apache.storm.starter.spout.RandomIntegerSpout)4 Yaml (org.yaml.snakeyaml.Yaml)4 JedisClusterConfig (org.apache.storm.redis.common.config.JedisClusterConfig)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 DruidBeamFactory (org.apache.storm.druid.bolt.DruidBeamFactory)2 ITupleDruidEventMapper (org.apache.storm.druid.bolt.ITupleDruidEventMapper)2 TupleDruidEventMapper (org.apache.storm.druid.bolt.TupleDruidEventMapper)2 EsConfig (org.apache.storm.elasticsearch.common.EsConfig)2