Examples with TestWordSpout org.apache.storm.testing.TestWordSpout used on opensource projects

Example 31 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class NettyIntegrationTest method testIntegration.

@Test
public void testIntegration() throws Exception {
    Map<String, Object> daemonConf = new HashMap<>();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, true);
    daemonConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    daemonConf.put(Config.STORM_MESSAGING_NETTY_AUTHENTICATION, false);
    daemonConf.put(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE, 1024000);
    daemonConf.put(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS, 1000);
    daemonConf.put(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS, 5000);
    daemonConf.put(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS, 1);
    daemonConf.put(Config.STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS, 1);
    Builder builder = new Builder().withSimulatedTime().withSupervisors(4).withSupervisorSlotPortMin(6710).withDaemonConf(daemonConf);
    try (LocalCluster cluster = builder.build()) {
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("1", new TestWordSpout(true), 4);
        topologyBuilder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1");
        StormTopology topology = topologyBuilder.createTopology();
        // important for test that tuples = multiple of 4 and 6
        List<FixedTuple> testTuples = Stream.of("a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b", "a", "b").map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, 3));
        completeTopologyParams.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}

Example 32 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class TopologyIntegrationTest method testMultiTasksPerCluster.

@Test
public void testMultiTasksPerCluster() throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(4).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true));
        builder.setBolt("2", new EmitTaskIdBolt(), 3).allGrouping("1").addConfigurations(Collections.singletonMap(Config.TOPOLOGY_TASKS, 6));
        StormTopology topology = builder.createTopology();
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", Collections.singletonList(new FixedTuple(new Values("a")))));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values(0), new Values(1), new Values(2), new Values(3), new Values(4), new Values(5)));
    }
}

Example 33 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class RollingTopWords method run.

/**
 * Submits (runs) the topology.
 *
 * <p>Usage: "RollingTopWords [topology-name] [-local]"
 *
 * <p>By default, the topology is run locally under the name
 * "slidingWindowCounts".
 *
 * <p>Examples:
 * ```
 * # Runs in remote/cluster mode, with topology name "production-topology"
 * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
 *
 * @param args
 *          First positional argument (optional) is topology name, second
 *          positional argument (optional) defines whether to run the topology
 *          locally ("-local") or remotely, i.e. on a real cluster.
 */
@Override
protected int run(String[] args) {
    String topologyName = "slidingWindowCounts";
    if (args.length >= 1) {
        topologyName = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String intermediateRankerId = "intermediateRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj"));
    String totalRankerId = "finalRanker";
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
    LOG.info("Topology name: " + topologyName);
    return submit(topologyName, conf, builder);
}