Examples with FeederSpout org.apache.storm.testing.FeederSpout used on opensource projects

Example 1 with FeederSpout

use of org.apache.storm.testing.FeederSpout in project open-kilda by telstra.

the class TickBoltTest method BasicTickTest.

@Test
public void BasicTickTest() throws IOException {
    System.out.println("==> Starting BasicTickTest");
    String spoutId = "feeder.spout";
    String boltId = "tick.bolt";
    String topoId = "TestTopology";
    TopologyBuilder builder = new TopologyBuilder();
    FeederSpout spout = TickBoltTest.createFeeder();
    builder.setSpout(spoutId, spout);
    SimpleStatefulTick tickBolt = new SimpleStatefulTick();
    builder.setBolt(boltId, tickBolt).shuffleGrouping(spoutId);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(topoId, TestUtils.stormConfig(), builder.createTopology());
    /* Let's Submit Stuff! */
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
    spout.feed(Arrays.asList(new String[] { "key1", "msg1" }));
    /* And sleep some more */
    Uninterruptibles.sleepUninterruptibly(6, TimeUnit.SECONDS);
    // TODO: this test isn't great .. the number of lines in the file from ticks could vary
    int expectedLines = 3;
    Assert.assertTrue("We should have at least " + expectedLines + " lines in the test file.", expectedLines <= tickBolt.tickFile.numLines());
    Assert.assertEquals(1, tickBolt.workFile.numLines());
    cluster.killTopology(topoId);
    Utils.sleep(4 * 1000);
}

Example 2 with FeederSpout

use of org.apache.storm.testing.FeederSpout in project storm by apache.

the class SingleJoinExample method main.

public static void main(String[] args) throws Exception {
    if (!NimbusClient.isLocalOverride()) {
        throw new IllegalStateException("This example only works in local mode.  " + "Run with storm local not storm jar");
    }
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("gender", genderSpout);
    builder.setSpout("age", ageSpout);
    builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")).fieldsGrouping("age", new Fields("id"));
    Config conf = new Config();
    conf.setDebug(true);
    StormSubmitter.submitTopology("join-example", conf, builder.createTopology());
    for (int i = 0; i < 10; i++) {
        String gender;
        if (i % 2 == 0) {
            gender = "male";
        } else {
            gender = "female";
        }
        genderSpout.feed(new Values(i, gender));
    }
    for (int i = 9; i >= 0; i--) {
        ageSpout.feed(new Values(i, i + 20));
    }
}

Example 3 with FeederSpout

use of org.apache.storm.testing.FeederSpout in project storm by apache.

the class TestingTest method testDisableTupleTimeout.

@Test
public void testDisableTupleTimeout() throws Exception {
    Config daemonConf = new Config();
    daemonConf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false);
    try (LocalCluster cluster = new LocalCluster.Builder().withDaemonConf(daemonConf).withSimulatedTime().build()) {
        FeederSpout feeder = new FeederSpout(new Fields("field1"));
        AckFailMapTracker tracker = new AckFailMapTracker();
        feeder.setAckFailDelegate(tracker);
        Map<String, Thrift.SpoutDetails> spoutMap = new HashMap<>();
        spoutMap.put("1", Thrift.prepareSpoutDetails(feeder));
        Map<String, Thrift.BoltDetails> boltMap = new HashMap<>();
        boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new AckEveryOtherBolt()));
        StormTopology topology = Thrift.buildTopology(spoutMap, boltMap);
        Config stormConf = new Config();
        stormConf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10);
        stormConf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false);
        cluster.submitTopology("disable-timeout-tester", stormConf, topology);
        feeder.feed(new Values("a"), 1);
        feeder.feed(new Values("b"), 2);
        feeder.feed(new Values("c"), 3);
        cluster.advanceClusterTime(9);
        assertAcked(tracker, 1, 3);
        assertThat(tracker.isFailed(2), is(false));
        cluster.advanceClusterTime(12);
        assertThat(tracker.isFailed(2), is(false));
    }
}

Example 4 with FeederSpout

use of org.apache.storm.testing.FeederSpout in project storm by apache.

the class TopologyIntegrationTest method testTimeout.

@Test
public void testTimeout() throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(4).withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)).build()) {
        FeederSpout feeder = new FeederSpout(new Fields("field1"));
        AckFailMapTracker tracker = new AckFailMapTracker();
        feeder.setAckFailDelegate(tracker);
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", feeder);
        builder.setBolt("2", new AckEveryOtherBolt()).globalGrouping("1");
        StormTopology topology = builder.createTopology();
        cluster.submitTopology("timeout-tester", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology);
        cluster.advanceClusterTime(11);
        feeder.feed(new Values("a"), 1);
        feeder.feed(new Values("b"), 2);
        feeder.feed(new Values("c"), 3);
        cluster.advanceClusterTime(9);
        assertAcked(tracker, 1, 3);
        assertThat(tracker.isFailed(2), is(false));
        cluster.advanceClusterTime(12);
        assertFailed(tracker, 2);
    }
}

Example 5 with FeederSpout

use of org.apache.storm.testing.FeederSpout in project storm by apache.

the class TopologyIntegrationTest method testSubmitInactiveTopology.

@Test
public void testSubmitInactiveTopology() throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)).build()) {
        FeederSpout feeder = new FeederSpout(new Fields("field1"));
        AckFailMapTracker tracker = new AckFailMapTracker();
        feeder.setAckFailDelegate(tracker);
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", feeder);
        builder.setSpout("2", new OpenTrackedSpout());
        builder.setBolt("3", new PrepareTrackedBolt()).globalGrouping("1");
        boltPrepared = false;
        spoutOpened = false;
        StormTopology topology = builder.createTopology();
        cluster.submitTopologyWithOpts("test", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology, new SubmitOptions(TopologyInitialStatus.INACTIVE));
        cluster.advanceClusterTime(11);
        feeder.feed(new Values("a"), 1);
        cluster.advanceClusterTime(9);
        assertThat(boltPrepared, is(false));
        assertThat(spoutOpened, is(false));
        cluster.getNimbus().activate("test");
        cluster.advanceClusterTime(12);
        assertAcked(tracker, 1);
        assertThat(boltPrepared, is(true));
        assertThat(spoutOpened, is(true));
    }
}