Examples with TestWordSpout backtype.storm.testing.TestWordSpout used on opensource projects

Example 1 with TestWordSpout

use of backtype.storm.testing.TestWordSpout in project storm-lib by xumingming.

the class TestingApiDemo method testBasicTopology.

public void testBasicTopology() {
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setSupervisors(4);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    mkClusterParam.setDaemonConf(daemonConf);
    /**
     * This is a combination of <code>Testing.withLocalCluster</code> and <code>Testing.withSimulatedTime</code>.
     */
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            // build the test topology
            TopologyBuilder builder = new TopologyBuilder();
            builder.setSpout("1", new TestWordSpout(true), 3);
            builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
            builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
            builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
            StormTopology topology = builder.createTopology();
            // complete the topology
            // prepare the mock data
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("1", new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan"));
            // prepare the config
            Config conf = new Config();
            conf.setNumWorkers(2);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            /**
             * TODO
             */
            Map result = Testing.completeTopology(cluster, topology, completeTopologyParam);
            // check whether the result is right
            assertTrue(Testing.multiseteq(new Values(new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan")), Testing.readTuples(result, "1")));
            assertTrue(Testing.multiseteq(new Values(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)), Testing.readTuples(result, "2")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "3")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "4")));
        }
    });
}

Example 2 with TestWordSpout

use of backtype.storm.testing.TestWordSpout in project jstorm by alibaba.

the class RollingTopWords method wireTopology.

private void wireTopology() throws InterruptedException {
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
}

Example 3 with TestWordSpout

use of backtype.storm.testing.TestWordSpout in project ud381 by udacity.

the class ReporterExclamationTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // ********* BEGIN stage2 exercise part 2-of-2 ***********
    // attach the word spout to the topology - parallelism of 10
    builder.setSpout("word", new TestWordSpout(), 10);
    // attach the exclamation bolt to the topology - parallelism of 3
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    // attach another exclamation bolt to the topology - parallelism of 2
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    // ********* END stage2 exercise part 2-of-2 ***********
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("exclamation", conf, builder.createTopology());
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // kill the topology
        cluster.killTopology("exclamation");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 4 with TestWordSpout

use of backtype.storm.testing.TestWordSpout in project jstorm by alibaba.

the class TaskInDifferentNodeTopology method test.

public static void test() throws Exception {
    JStormHelper.cleanCluster();
    hosts = JStormHelper.getSupervisorHosts();
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 1);
    /**
     *******
     *
     * This make sure the tasks will run on different nodes
     */
    Map<String, Object> componentMap = new HashMap<>();
    ConfigExtension.setTaskOnDifferentNode(componentMap, true);
    builder.setBolt(BOLT_NAME, new ExclamationLoggingBolt(), hosts.size()).localFirstGrouping("word").addConfigurations(componentMap);
    if (isLocal == false) {
        if (spoutSingle == true) {
            conf.setNumWorkers(hosts.size() + 3);
        } else {
            conf.setNumWorkers(hosts.size());
        }
    }
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 180, new Validator(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 5 with TestWordSpout

use of backtype.storm.testing.TestWordSpout in project jstorm by alibaba.

the class SkewedRollingTopWords method wireTopology.

private void wireTopology() throws InterruptedException {
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String aggId = "aggregator";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
    builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
}