Examples with TopologyBuilder backtype.storm.topology.TopologyBuilder used on opensource projects

Example 26 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class FastWordCountEventTimeWindowTopology method test.

public static void test() throws Exception {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.eventTimeWindow(Time.milliseconds(3L)).withTimestampExtractor(wordCountBolt).withWatermarkGenerator(new PeriodicWatermarkGenerator(Time.milliseconds(1L), Time.milliseconds(10L))), count_Parallelism_hint).fieldsGrouping("spout", new Fields("word", "ts"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
}

Example 27 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class FastWordCountSessionEventTimeWindowTopology method test.

public static void test() throws Exception {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L)).withTimestampExtractor(wordCountBolt).withWindowStateMerger(wordCountBolt), count_Parallelism_hint).fieldsGrouping("spout", new Fields("word", "ts"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
}

Example 28 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class FastWordCountTimeWindowTopology method test.

public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    boolean isLocalShuffle = JStormUtils.parseBoolean(conf.get("is.local.first.group"), false);
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    if (isLocalShuffle) {
        builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localFirstGrouping("spout");
    } else {
        builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    }
    builder.setBolt("count", new WordCount().timeWindow(Time.seconds(1L), Time.milliseconds(500L)).withStateSize(Time.hours(2)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 29 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SequenceTopologyTest method testSequenceTopology.

@Test
public void testSequenceTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceTestSpout(), SPOUT_PARALLELISM_HINT);
    topologyBuilder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SequenceTestSplitRecord(), BOLT_PARALLELISM_HINT).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    topologyBuilder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new SequenceTestPairCount(), BOLT_PARALLELISM_HINT).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
    topologyBuilder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new SequenceTestPairCount(), BOLT_PARALLELISM_HINT).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
    topologyBuilder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new SequenceTestMergeRecord(), BOLT_PARALLELISM_HINT).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
    topologyBuilder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new SequenceTestTotalCount(), BOLT_PARALLELISM_HINT).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    // use config in detail.yaml
    Map conf = new HashMap();
    // Config.setFallBackOnJavaSerialization(conf, true);      //fall.back.on.java.serialization: true
    // //enable.split: true
    // Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
    // Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    Config.setNumAckers(conf, 1);
    Config.setNumWorkers(conf, 3);
    // set a limit for the spout to get a precise
    conf.put("spout.max.sending.num", SPOUT_MAX_SEND_NUM);
    // number to make sure the topology works well.
    conf.put(Config.TOPOLOGY_NAME, "SequenceTopologyTest");
    // the following is just for the JStormUnitTestMetricValidator to pick the metric data
    // from all the metrics.If you are not using JStormUnitTestMetricValidator, it is useless.
    // The first element is the key that register in the metric, the second one is the key
    // map with the metric value as a parameter in the callback function validateMetrics().
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_SUCCESS);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_FAIL);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_TRADE_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_CUSTOMER_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPLIT_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_PAIR_TRADE_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_PAIR_CUSTOMER_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_MERGE_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_EXECUTE);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_TRADE_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_CUSTOMER_SUM);
    JStormUnitTestMetricValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            for (Map.Entry<String, Double> entry : metrics.entrySet()) LOG.info("user define metric Key = " + entry.getKey() + " Value = " + entry.getValue());
            int spoutEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_EMIT)).doubleValue();
            int spoutSuccess = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_SUCCESS)).doubleValue();
            int spoutFail = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_FAIL)).doubleValue();
            long spoutTradeSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_TRADE_SUM)).doubleValue();
            long spoutCustomerSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_CUSTOMER_SUM)).doubleValue();
            int splitEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPLIT_EMIT)).doubleValue();
            int pairTradeEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_PAIR_TRADE_EMIT)).doubleValue();
            int pairCustomerEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_PAIR_CUSTOMER_EMIT)).doubleValue();
            int mergeEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_MERGE_EMIT)).doubleValue();
            int totalExecute = (int) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_EXECUTE)).doubleValue();
            long totalTradeSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_TRADE_SUM)).doubleValue();
            long totalCustomerSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_CUSTOMER_SUM)).doubleValue();
            assertEquals(SPOUT_MAX_SEND_NUM, spoutEmit);
            assertEquals(spoutEmit, spoutSuccess);
            assertEquals(0, spoutFail);
            assertEquals(2 * spoutEmit, splitEmit);
            assertEquals(splitEmit, pairTradeEmit * 2);
            assertEquals(splitEmit, pairCustomerEmit * 2);
            assertEquals(splitEmit, mergeEmit * 2);
            assertEquals(mergeEmit, totalExecute);
            assertEquals(spoutTradeSum, totalTradeSum);
            assertEquals(spoutCustomerSum, totalCustomerSum);
            return true;
        }
    };
    // the below line time in second 150 is recommend, at least it should be more than 120 since the
    // metric data was grabbed every 60s but not so precise.
    boolean result = JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), conf, 150, validator);
    assertTrue("Topology should pass the validator", result);
}

Example 30 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class RollingTopWordsTest method testRollingTopWords.

@Test
public void testRollingTopWords() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("windowTestWordSpout", new WindowTestWordSpout(), 5);
    topologyBuilder.setBolt("windowTestRollingCountBolt", new WindowTestRollingCountBolt(9, 3), 4).fieldsGrouping("windowTestWordSpout", new Fields("word")).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    topologyBuilder.setBolt("windowTestIntermediateRankingBolt", new WindowTestIntermediateRankingBolt(DEFAULT_COUNT), 4).fieldsGrouping("windowTestRollingCountBolt", new Fields("obj"));
    topologyBuilder.setBolt("windowTestTotalRankingsBolt", new WindowTestTotalRankingsBolt(DEFAULT_COUNT)).globalGrouping("windowTestIntermediateRankingBolt");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "RollingTopWordsTest");
    // I really don't know how to validate if the result is right since
    // the tick time is not precise. It makes the output after passing
    // a window is unpredictable.
    // Now I just let it pass all the time.
    // TODO:FIX ME: how to validate if the result is right?
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 90, null);
}