Examples with TestWordSpout org.apache.storm.testing.TestWordSpout used on opensource projects

Example 16 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class TestResourceAwareScheduler method testMemoryLoadLargerThanMaxHeapSize.

@Test(expected = IllegalArgumentException.class)
public void testMemoryLoadLargerThanMaxHeapSize() throws Exception {
    // Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0,
    // Largest memory requirement of a component in the topology).
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 4);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(defaultTopologyConf);
    config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 129.0);
    StormSubmitter.submitTopologyWithProgressBar("test", config1, stormTopology1);
}

Example 17 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class MultipleLoggerTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(2);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology())) {
            Utils.sleep(10000);
        }
    }
}

Example 18 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class ResourceAwareExampleTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 5);
    //set cpu requirement
    spout.setCPULoad(20);
    //set onheap and offheap memory requirement
    spout.setMemoryLoad(64, 16);
    BoltDeclarer bolt1 = builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    //sets cpu requirement.  Not neccessary to set both CPU and memory.
    //For requirements not set, a default value will be used
    bolt1.setCPULoad(15);
    BoltDeclarer bolt2 = builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    bolt2.setMemoryLoad(100);
    Config conf = new Config();
    conf.setDebug(true);
    /**
     * Use to limit the maximum amount of memory (in MB) allocated to one worker process.
     * Can be used to spread executors to to multiple workers
     */
    conf.setTopologyWorkerMaxHeapSize(1024.0);
    //topology priority describing the importance of the topology in decreasing importance starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
    //Recommended range of 0-29 but no hard limit set.
    conf.setTopologyPriority(29);
    // Set strategy to schedule topology. If not specified, default to org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy
    conf.setTopologyStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology())) {
            Utils.sleep(10000);
        }
    }
}

Example 19 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class RollingTopWords method run.

/**
   * Submits (runs) the topology.
   *
   * Usage: "RollingTopWords [topology-name] [-local]"
   *
   * By default, the topology is run locally under the name
   * "slidingWindowCounts".
   *
   * Examples:
   *
   * ```
   *
   * # Runs in local mode (LocalCluster), with topology name "slidingWindowCounts"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords -local
   * 
   * # Runs in local mode (LocalCluster), with topology name "foobar"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords foobar -local
   * 
   * # Runs in local mode (LocalCluster) for 30 seconds, with topology name "foobar" 
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords foobar -local -ttl 30
   *
   * # Runs in remote/cluster mode, with topology name "production-topology"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
   *
   * @param args
   *          First positional argument (optional) is topology name, second
   *          positional argument (optional) defines whether to run the topology
   *          locally ("-local") or remotely, i.e. on a real cluster.
   * @throws Exception
   */
protected int run(String[] args) {
    String topologyName = "slidingWindowCounts";
    if (args.length >= 1) {
        topologyName = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
    LOG.info("Topology name: " + topologyName);
    return submit(topologyName, conf, builder);
}