Examples with TopologyBuilder backtype.storm.topology.TopologyBuilder used on opensource projects

Example 31 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SlidingTupleTsTopology method test.

public static void test() {
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        TopologyBuilder builder = new TopologyBuilder();
        BaseWindowedBolt bolt = new SlidingWindowSumBolt().withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)).withTimestampField("ts").withLag(new Duration(5, TimeUnit.SECONDS));
        builder.setSpout("integer", new RandomIntegerSpout(), 1);
        builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
        builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
        conf.setDebug(true);
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 32 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class TickTupleTest method testTickTuple.

@Test
public void testTickTuple() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    // the spout is useless in this case, all the methods are empty
    topologyBuilder.setSpout("spout", new TickTupleTestSpout());
    // note that there is no grouping here, the bolt should not receive any tuple from spout
    // I increase the parallelism of the bolt to check if it is correct when we have a
    // parallelism greater than 1.
    topologyBuilder.setBolt("bolt", new TickTupleTestBolt(), TICK_TUPLE_BOLT_PARALLELISM).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, TICK_TUPLE_CYCLE);
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("TickTupleTest.TickMeter");
    userDefineMetrics.add("TickTupleTest.NonTickCounter");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            // there is $TICK_TUPLE_BOLT_PARALLELISM bolts, so the TickMeter need divided by $TICK_TUPLE_BOLT_PARALLELISM
            double cycle = 1 / (metrics.get("TickTupleTest.TickMeter") / TICK_TUPLE_BOLT_PARALLELISM);
            LOG.info("TickTupleTest.TickMeter = " + metrics.get("TickTupleTest.TickMeter"));
            LOG.info("Tick cycle  = " + cycle);
            assertTrue("The tick cycle should be in range of 0.8*TICK_TUPLE_CYCLE and 1.2*TICK_TUPLE_CYCLE", cycle > 0.9f * TICK_TUPLE_CYCLE && cycle < 1.1f * TICK_TUPLE_CYCLE);
            assertEquals(0, (int) (metrics.get("TickTupleTest.NonTickCounter").doubleValue()));
            return true;
        }
    };
    Config config = new Config();
    config.put(Config.TOPOLOGY_NAME, "TickTupleTest");
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}

Example 33 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class InOrderDeliveryTest method testInOrderDelivery.

@Test
public void testInOrderDelivery() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("spout", new InOrderTestSpout(SPOUT_MAX_SEND_NUM), SPOUT_PARALLELISM_HINT);
    // note that here we use fieldsGrouping, so that the tuples with the same "c1" will be sent to the same bolt.
    // as a result, even though we have BOLT_PARALLELISM_HINT bolts, each bolt maintains a order of tuples from
    // spouts that it can receive.
    topologyBuilder.setBolt("bolt", new InOrderTestBolt(), BOLT_PARALLELISM_HINT).fieldsGrouping("spout", new Fields("c1"));
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "InOrderDeliveryTest");
    config.put("topology.debug.metric.names", "emit,success,fail");
    config.put("topology.debug", false);
    config.put("topology.enable.metric.debug", true);
    // the following is just for the JStormUnitTestMetricValidator to pick the metric data
    // from all the metrics.If you are not using JStormUnitTestMetricValidator, it is useless.
    // The element is the key map with the metric value as a parameter in the callback
    // function validateMetrics().
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_SPOUT_EMIT);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_FAIL);
    JStormUnitTestMetricValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutEmit = (int) metrics.get(InOrderTestMetricsDef.METRIC_SPOUT_EMIT).doubleValue();
            int boltSuccess = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS).doubleValue();
            int boltFail = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_FAIL).doubleValue();
            LOG.info("validateMetrics: " + "spout emit = " + spoutEmit + " bolt success = " + boltSuccess);
            assertEquals(SPOUT_MAX_SEND_NUM * SPOUT_PARALLELISM_HINT, spoutEmit);
            // all tuples should be in order
            assertEquals(spoutEmit, boltSuccess);
            assertEquals(0, boltFail);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}

Example 34 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class TridentReach method test.

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    conf.setMaxSpoutPending(20);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    try {
        JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 35 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SkewedRollingTopWordsTest method testSkewedRollingTopWords.

@Test
public void testSkewedRollingTopWords() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("windowTestWordSpout", new WindowTestWordSpout(), 5);
    topologyBuilder.setBolt("windowTestRollingCountBolt", new WindowTestRollingCountBolt(9, 3), 4).partialKeyGrouping("windowTestWordSpout", new Fields("word"));
    topologyBuilder.setBolt("windowTestCountAggBolt", new WindowTestCountAggBolt(), 4).fieldsGrouping("windowTestRollingCountBolt", new Fields("obj"));
    topologyBuilder.setBolt("windowTestIntermediateRankingBolt", new WindowTestIntermediateRankingBolt(DEFAULT_COUNT), 4).fieldsGrouping("windowTestCountAggBolt", new Fields("obj"));
    topologyBuilder.setBolt("windowTestTotalRankingsBolt", new WindowTestTotalRankingsBolt(DEFAULT_COUNT)).globalGrouping("windowTestIntermediateRankingBolt");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "SkewedRollingTopWordsTest");
    // I really don't know how to validate if the result is right since
    // the tick time is not precise. It makes the output after passing
    // a window is unpredictable.
    // Now I just let it pass all the time.
    // TODO:FIX ME: how to validate if the result is right?
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 90, null);
}