Examples with LocalTopology org.apache.storm.LocalCluster.LocalTopology used on opensource projects

Example 41 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class TridentHiveTopology method main.

public static void main(String[] args) throws Exception {
    String metaStoreURI = args[0];
    String dbName = args[1];
    String tblName = args[2];
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 3) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("tridentHiveTopology", conf, buildTopology(metaStoreURI, dbName, tblName, null, null))) {
            LOG.info("waiting for 60 seconds");
            waitForSeconds(60);
        }
        System.exit(0);
    } else if (args.length == 4) {
        try {
            StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName, null, null));
        } catch (SubmitterHookException e) {
            LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
        } catch (Exception e) {
            LOG.warn("Failed to submit topology ", e);
        }
    } else if (args.length == 6) {
        try {
            StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName, args[4], args[5]));
        } catch (SubmitterHookException e) {
            LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
        } catch (Exception e) {
            LOG.warn("Failed to submit topology ", e);
        }
    } else {
        LOG.info("Usage: TridentHiveTopology metastoreURI dbName tableName [topologyNamey]");
    }
}

Example 42 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class PersistentWordCount method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    String host = TEST_REDIS_HOST;
    int port = TEST_REDIS_PORT;
    if (args.length >= 2) {
        host = args[0];
        port = Integer.parseInt(args[1]);
    }
    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder().setHost(host).setPort(port).build();
    WordSpout spout = new WordSpout();
    WordCounter bolt = new WordCounter();
    RedisStoreMapper storeMapper = setupStoreMapper();
    RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);
    // wordSpout ==> countBolt ==> RedisBolt
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(COUNT_BOLT, bolt, 1).fieldsGrouping(WORD_SPOUT, new Fields("word"));
    builder.setBolt(STORE_BOLT, storeBolt, 1).shuffleGrouping(COUNT_BOLT);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: PersistentWordCount <redis host> <redis port> (topology name)");
    }
}

Example 43 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class WhitelistWordCount method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    String host = TEST_REDIS_HOST;
    int port = TEST_REDIS_PORT;
    if (args.length >= 2) {
        host = args[0];
        port = Integer.parseInt(args[1]);
    }
    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder().setHost(host).setPort(port).build();
    WordSpout spout = new WordSpout();
    RedisFilterMapper filterMapper = setupWhitelistMapper();
    RedisFilterBolt whitelistBolt = new RedisFilterBolt(poolConfig, filterMapper);
    WordCounter wordCounterBolt = new WordCounter();
    PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt();
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(WHITELIST_BOLT, whitelistBolt, 1).shuffleGrouping(WORD_SPOUT);
    builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT, new Fields("word"));
    builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(COUNT_BOLT);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: WhitelistWordCount <redis host> <redis port> (topology name)");
    }
}

Example 44 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class WordCountTridentRedisCluster method main.

public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        System.out.println("Usage: WordCountTrident 0(storm-local)|1(storm-cluster) 127.0.0.1:6379,127.0.0.1:6380");
        System.exit(1);
    }
    Integer flag = Integer.valueOf(args[0]);
    String redisHostPort = args[1];
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (flag == 0) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort))) {
            Thread.sleep(60 * 1000);
        }
        System.exit(0);
    } else if (flag == 1) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));
    } else {
        System.out.println("Usage: WordCountTrident 0(storm-local)|1(storm-cluster) redis-host redis-port");
    }
}

Example 45 with LocalTopology

use of org.apache.storm.LocalCluster.LocalTopology in project storm by apache.

the class UpdateWordCount method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    String url = TEST_MONGODB_URL;
    String collectionName = TEST_MONGODB_COLLECTION_NAME;
    if (args.length >= 2) {
        url = args[0];
        collectionName = args[1];
    }
    WordSpout spout = new WordSpout();
    WordCounter bolt = new WordCounter();
    MongoUpdateMapper mapper = new SimpleMongoUpdateMapper().withFields("word", "count");
    QueryFilterCreator filterCreator = new SimpleQueryFilterCreator().withField("word");
    MongoUpdateBolt updateBolt = new MongoUpdateBolt(url, collectionName, filterCreator, mapper);
    //if a new document should be inserted if there are no matches to the query filter
    //updateBolt.withUpsert(true);
    //whether find all documents according to the query filter
    //updateBolt.withMany(true);
    // wordSpout ==> countBolt ==> MongoUpdateBolt
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT);
    builder.setBolt(UPDATE_BOLT, updateBolt, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: UpdateWordCount <mongodb url> <mongodb collection> [topology name]");
    }
}