Examples with LocalCluster org.apache.storm.LocalCluster used on opensource projects

Example 56 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class TridentFileTopology method main.

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    conf.put("hdfs.config", yamlConf);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0]))) {
            Thread.sleep(120 * 1000);
        }
    } else if (args.length == 3) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[2], conf, buildTopology(args[0]));
    } else {
        System.out.println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}

Example 57 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class TridentSequenceTopology method main.

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    conf.put("hdfs.config", yamlConf);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("wordCounter", conf, buildTopology(args[0]))) {
            Thread.sleep(120 * 1000);
        }
    } else if (args.length == 3) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[2], conf, buildTopology(args[0]));
    } else {
        System.out.println("Usage: TridentSequenceTopology [hdfs url] [hdfs yaml config file] <topology name>");
    }
}

Example 58 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class BucketTestHiveTopology method main.

public static void main(String[] args) throws Exception {
    if ((args == null) || (args.length < 7)) {
        System.out.println("Usage: BucketTestHiveTopology metastoreURI " + "dbName tableName dataFileLocation hiveBatchSize " + "hiveTickTupl]eIntervalSecs workers  [topologyNamey] [keytab file]" + " [principal name] ");
        System.exit(1);
    }
    String metaStoreURI = args[0];
    String dbName = args[1];
    String tblName = args[2];
    String sourceFileLocation = args[3];
    Integer hiveBatchSize = Integer.parseInt(args[4]);
    Integer hiveTickTupleIntervalSecs = Integer.parseInt(args[5]);
    Integer workers = Integer.parseInt(args[6]);
    String[] colNames = { "ss_sold_date_sk", "ss_sold_time_sk", "ss_item_sk", "ss_customer_sk", "ss_cdemo_sk", "ss_hdemo_sk", "ss_addr_sk", "ss_store_sk", "ss_promo_sk", "ss_ticket_number", "ss_quantity", "ss_wholesale_cost", "ss_list_price", "ss_sales_price", "ss_ext_discount_amt", "ss_ext_sales_price", "ss_ext_wholesale_cost", "ss_ext_list_price", "ss_ext_tax", "ss_coupon_amt", "ss_net_paid", "ss_net_paid_inc_tax", "ss_net_profit" };
    Config config = new Config();
    config.setNumWorkers(workers);
    UserDataSpout spout = new UserDataSpout().withDataFile(sourceFileLocation);
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withTimeAsPartitionField("yyyy/MM/dd");
    HiveOptions hiveOptions;
    hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(hiveBatchSize);
    // had to make tick tuple a mandatory argument since its positional
    if (hiveTickTupleIntervalSecs > 0) {
        hiveOptions.withTickTupleInterval(hiveTickTupleIntervalSecs);
    }
    if (args.length == 10) {
        hiveOptions.withKerberosKeytab(args[8]).withKerberosPrincipal(args[9]);
    }
    HiveBolt hiveBolt = new HiveBolt(hiveOptions);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(USER_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, hiveBolt, 14).shuffleGrouping(USER_SPOUT_ID);
    if (args.length == 6) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology())) {
            waitForSeconds(20);
        }
        System.exit(0);
    } else {
        StormSubmitter.submitTopology(args[7], config, builder.createTopology());
    }
}

Example 59 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class TridentHiveTopology method main.

public static void main(String[] args) throws Exception {
    String metaStoreURI = args[0];
    String dbName = args[1];
    String tblName = args[2];
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 3) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("tridentHiveTopology", conf, buildTopology(metaStoreURI, dbName, tblName, null, null))) {
            LOG.info("waiting for 60 seconds");
            waitForSeconds(60);
        }
        System.exit(0);
    } else if (args.length == 4) {
        try {
            StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName, null, null));
        } catch (SubmitterHookException e) {
            LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
        } catch (Exception e) {
            LOG.warn("Failed to submit topology ", e);
        }
    } else if (args.length == 6) {
        try {
            StormSubmitter.submitTopology(args[3], conf, buildTopology(metaStoreURI, dbName, tblName, args[4], args[5]));
        } catch (SubmitterHookException e) {
            LOG.warn("Topology is submitted but invoking ISubmitterHook failed", e);
        } catch (Exception e) {
            LOG.warn("Failed to submit topology ", e);
        }
    } else {
        LOG.info("Usage: TridentHiveTopology metastoreURI dbName tableName [topologyNamey]");
    }
}

Example 60 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class PersistentWordCount method main.

public static void main(String[] args) throws Exception {
    Config config = new Config();
    String host = TEST_REDIS_HOST;
    int port = TEST_REDIS_PORT;
    if (args.length >= 2) {
        host = args[0];
        port = Integer.parseInt(args[1]);
    }
    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder().setHost(host).setPort(port).build();
    WordSpout spout = new WordSpout();
    WordCounter bolt = new WordCounter();
    RedisStoreMapper storeMapper = setupStoreMapper();
    RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);
    // wordSpout ==> countBolt ==> RedisBolt
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(COUNT_BOLT, bolt, 1).fieldsGrouping(WORD_SPOUT, new Fields("word"));
    builder.setBolt(STORE_BOLT, storeBolt, 1).shuffleGrouping(COUNT_BOLT);
    if (args.length == 2) {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", config, builder.createTopology())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[2], config, builder.createTopology());
    } else {
        System.out.println("Usage: PersistentWordCount <redis host> <redis port> (topology name)");
    }
}