Examples with JStormUnitTestMetricValidator com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator used on opensource projects

Example 1 with JStormUnitTestMetricValidator

use of com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator in project jstorm by alibaba.

the class SequenceTopologyTest method testSequenceTopology.

@Test
public void testSequenceTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceTestSpout(), SPOUT_PARALLELISM_HINT);
    topologyBuilder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SequenceTestSplitRecord(), BOLT_PARALLELISM_HINT).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    topologyBuilder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new SequenceTestPairCount(), BOLT_PARALLELISM_HINT).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
    topologyBuilder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new SequenceTestPairCount(), BOLT_PARALLELISM_HINT).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
    topologyBuilder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new SequenceTestMergeRecord(), BOLT_PARALLELISM_HINT).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
    topologyBuilder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new SequenceTestTotalCount(), BOLT_PARALLELISM_HINT).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    // use config in detail.yaml
    Map conf = new HashMap();
    // Config.setFallBackOnJavaSerialization(conf, true);      //fall.back.on.java.serialization: true
    // //enable.split: true
    // Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
    // Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    Config.setNumAckers(conf, 1);
    Config.setNumWorkers(conf, 3);
    // set a limit for the spout to get a precise
    conf.put("spout.max.sending.num", SPOUT_MAX_SEND_NUM);
    // number to make sure the topology works well.
    conf.put(Config.TOPOLOGY_NAME, "SequenceTopologyTest");
    // the following is just for the JStormUnitTestMetricValidator to pick the metric data
    // from all the metrics.If you are not using JStormUnitTestMetricValidator, it is useless.
    // The first element is the key that register in the metric, the second one is the key
    // map with the metric value as a parameter in the callback function validateMetrics().
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_SUCCESS);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_FAIL);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_TRADE_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPOUT_CUSTOMER_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_SPLIT_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_PAIR_TRADE_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_PAIR_CUSTOMER_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_MERGE_EMIT);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_EXECUTE);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_TRADE_SUM);
    userDefineMetrics.add(SequenceTestMetricsDef.METRIC_TOTAL_CUSTOMER_SUM);
    JStormUnitTestMetricValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            for (Map.Entry<String, Double> entry : metrics.entrySet()) LOG.info("user define metric Key = " + entry.getKey() + " Value = " + entry.getValue());
            int spoutEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_EMIT)).doubleValue();
            int spoutSuccess = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_SUCCESS)).doubleValue();
            int spoutFail = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_FAIL)).doubleValue();
            long spoutTradeSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_TRADE_SUM)).doubleValue();
            long spoutCustomerSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_SPOUT_CUSTOMER_SUM)).doubleValue();
            int splitEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_SPLIT_EMIT)).doubleValue();
            int pairTradeEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_PAIR_TRADE_EMIT)).doubleValue();
            int pairCustomerEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_PAIR_CUSTOMER_EMIT)).doubleValue();
            int mergeEmit = (int) (metrics.get(SequenceTestMetricsDef.METRIC_MERGE_EMIT)).doubleValue();
            int totalExecute = (int) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_EXECUTE)).doubleValue();
            long totalTradeSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_TRADE_SUM)).doubleValue();
            long totalCustomerSum = (long) (metrics.get(SequenceTestMetricsDef.METRIC_TOTAL_CUSTOMER_SUM)).doubleValue();
            assertEquals(SPOUT_MAX_SEND_NUM, spoutEmit);
            assertEquals(spoutEmit, spoutSuccess);
            assertEquals(0, spoutFail);
            assertEquals(2 * spoutEmit, splitEmit);
            assertEquals(splitEmit, pairTradeEmit * 2);
            assertEquals(splitEmit, pairCustomerEmit * 2);
            assertEquals(splitEmit, mergeEmit * 2);
            assertEquals(mergeEmit, totalExecute);
            assertEquals(spoutTradeSum, totalTradeSum);
            assertEquals(spoutCustomerSum, totalCustomerSum);
            return true;
        }
    };
    // the below line time in second 150 is recommend, at least it should be more than 120 since the
    // metric data was grabbed every 60s but not so precise.
    boolean result = JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), conf, 150, validator);
    assertTrue("Topology should pass the validator", result);
}

Example 2 with JStormUnitTestMetricValidator

use of com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator in project jstorm by alibaba.

the class TickTupleTest method testTickTuple.

@Test
public void testTickTuple() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    // the spout is useless in this case, all the methods are empty
    topologyBuilder.setSpout("spout", new TickTupleTestSpout());
    // note that there is no grouping here, the bolt should not receive any tuple from spout
    // I increase the parallelism of the bolt to check if it is correct when we have a
    // parallelism greater than 1.
    topologyBuilder.setBolt("bolt", new TickTupleTestBolt(), TICK_TUPLE_BOLT_PARALLELISM).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, TICK_TUPLE_CYCLE);
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("TickTupleTest.TickMeter");
    userDefineMetrics.add("TickTupleTest.NonTickCounter");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            // there is $TICK_TUPLE_BOLT_PARALLELISM bolts, so the TickMeter need divided by $TICK_TUPLE_BOLT_PARALLELISM
            double cycle = 1 / (metrics.get("TickTupleTest.TickMeter") / TICK_TUPLE_BOLT_PARALLELISM);
            LOG.info("TickTupleTest.TickMeter = " + metrics.get("TickTupleTest.TickMeter"));
            LOG.info("Tick cycle  = " + cycle);
            assertTrue("The tick cycle should be in range of 0.8*TICK_TUPLE_CYCLE and 1.2*TICK_TUPLE_CYCLE", cycle > 0.9f * TICK_TUPLE_CYCLE && cycle < 1.1f * TICK_TUPLE_CYCLE);
            assertEquals(0, (int) (metrics.get("TickTupleTest.NonTickCounter").doubleValue()));
            return true;
        }
    };
    Config config = new Config();
    config.put(Config.TOPOLOGY_NAME, "TickTupleTest");
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}

Example 3 with JStormUnitTestMetricValidator

use of com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator in project jstorm by alibaba.

the class InOrderDeliveryTest method testInOrderDelivery.

@Test
public void testInOrderDelivery() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("spout", new InOrderTestSpout(SPOUT_MAX_SEND_NUM), SPOUT_PARALLELISM_HINT);
    // note that here we use fieldsGrouping, so that the tuples with the same "c1" will be sent to the same bolt.
    // as a result, even though we have BOLT_PARALLELISM_HINT bolts, each bolt maintains a order of tuples from
    // spouts that it can receive.
    topologyBuilder.setBolt("bolt", new InOrderTestBolt(), BOLT_PARALLELISM_HINT).fieldsGrouping("spout", new Fields("c1"));
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "InOrderDeliveryTest");
    config.put("topology.debug.metric.names", "emit,success,fail");
    config.put("topology.debug", false);
    config.put("topology.enable.metric.debug", true);
    // the following is just for the JStormUnitTestMetricValidator to pick the metric data
    // from all the metrics.If you are not using JStormUnitTestMetricValidator, it is useless.
    // The element is the key map with the metric value as a parameter in the callback
    // function validateMetrics().
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_SPOUT_EMIT);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_FAIL);
    JStormUnitTestMetricValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutEmit = (int) metrics.get(InOrderTestMetricsDef.METRIC_SPOUT_EMIT).doubleValue();
            int boltSuccess = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS).doubleValue();
            int boltFail = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_FAIL).doubleValue();
            LOG.info("validateMetrics: " + "spout emit = " + spoutEmit + " bolt success = " + boltSuccess);
            assertEquals(SPOUT_MAX_SEND_NUM * SPOUT_PARALLELISM_HINT, spoutEmit);
            // all tuples should be in order
            assertEquals(spoutEmit, boltSuccess);
            assertEquals(0, boltFail);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}

Example 4 with JStormUnitTestMetricValidator

use of com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator in project jstorm by alibaba.

the class SlidingTupleTsTopologyTest method testSlidingTupleTsTopology.

@Test
public void testSlidingTupleTsTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingTupleTestBolt().withWindow(new BaseWindowedBolt.Duration(WINDOW_LENGTH_SEC, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(WINDOW_SLIDE_SEC, TimeUnit.SECONDS)).withTimestampField("ts").withLag(new BaseWindowedBolt.Duration(WINDOW_LAG_SEC, TimeUnit.SECONDS));
    topologyBuilder.setSpout("spout", new SlidingTupleTestRandomSpout(SPOUT_LIMIT), 1);
    topologyBuilder.setBolt("sum", bolt, 1).shuffleGrouping("spout");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "SlidingTupleTsTopologyTest");
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("SlidingTupleTsTopologyTest.SpoutSum");
    userDefineMetrics.add("SlidingTupleTsTopologyTest.BoltSum");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutSum = (int) metrics.get("SlidingTupleTsTopologyTest.SpoutSum").doubleValue();
            int boltSum = (int) metrics.get("SlidingTupleTsTopologyTest.BoltSum").doubleValue();
            assertEquals(spoutSum, boltSum);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}

Example 5 with JStormUnitTestMetricValidator

use of com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator in project jstorm by alibaba.

the class SlidingWindowTopologyTest method testSlidingWindowTopology.

@Test
public void testSlidingWindowTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("spout", new SlidingWindowTestRandomSpout(SPOUT_LIMIT), 1);
    // the following bolt sums all the elements in the window. The window has length of 30 elements
    // and slide every 10 elements.
    // for example, if the spout generate 1, 2, 3, 4 ... then the SumBolt generate 55, 210, 465, 765 ...
    topologyBuilder.setBolt("sum", new SlidingWindowTestSumBolt().withWindow(new BaseWindowedBolt.Count(SUM_BOLT_WINDOW_LENGTH), new BaseWindowedBolt.Count(SUM_BOLT_WINDOW_SLIDE)), 1).shuffleGrouping("spout");
    // the following bolt calculate the average value of elements in the window. The window has length
    // of 3. So it generates the average of 3 elements and then wait for another 3 elements.
    topologyBuilder.setBolt("avg", new SlidingWindowTestAvgBolt().withTumblingWindow(new BaseWindowedBolt.Count(AVG_BOLT_WINDOW_LENGTH)), 1).shuffleGrouping("sum");
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("SlidingWindowTopologyTest.SpoutAvgSum");
    userDefineMetrics.add("SlidingWindowTopologyTest.BoltAvgSum");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "SlidingWindowTopologyTest");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutAvgSum = (int) metrics.get("SlidingWindowTopologyTest.SpoutAvgSum").doubleValue();
            int boltAvgSum = (int) metrics.get("SlidingWindowTopologyTest.BoltAvgSum").doubleValue();
            System.out.println(spoutAvgSum + " " + boltAvgSum);
            assertEquals(spoutAvgSum, boltAvgSum);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}