Search in sources :

Example 86 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SlidingTupleTsTopologyTest method testSlidingTupleTsTopology.

@Test
public void testSlidingTupleTsTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingTupleTestBolt().withWindow(new BaseWindowedBolt.Duration(WINDOW_LENGTH_SEC, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(WINDOW_SLIDE_SEC, TimeUnit.SECONDS)).withTimestampField("ts").withLag(new BaseWindowedBolt.Duration(WINDOW_LAG_SEC, TimeUnit.SECONDS));
    topologyBuilder.setSpout("spout", new SlidingTupleTestRandomSpout(SPOUT_LIMIT), 1);
    topologyBuilder.setBolt("sum", bolt, 1).shuffleGrouping("spout");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "SlidingTupleTsTopologyTest");
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("SlidingTupleTsTopologyTest.SpoutSum");
    userDefineMetrics.add("SlidingTupleTsTopologyTest.BoltSum");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutSum = (int) metrics.get("SlidingTupleTsTopologyTest.SpoutSum").doubleValue();
            int boltSum = (int) metrics.get("SlidingTupleTsTopologyTest.BoltSum").doubleValue();
            assertEquals(spoutSum, boltSum);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}
Also used : JStormUnitTestValidator(com.jstorm.example.unittests.utils.JStormUnitTestValidator) TopologyBuilder(backtype.storm.topology.TopologyBuilder) JStormUnitTestMetricValidator(com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator) BaseWindowedBolt(backtype.storm.topology.base.BaseWindowedBolt) Test(org.junit.Test)

Example 87 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SlidingWindowTopologyTest method testSlidingWindowTopology.

@Test
public void testSlidingWindowTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("spout", new SlidingWindowTestRandomSpout(SPOUT_LIMIT), 1);
    // the following bolt sums all the elements in the window. The window has length of 30 elements
    // and slide every 10 elements.
    // for example, if the spout generate 1, 2, 3, 4 ... then the SumBolt generate 55, 210, 465, 765 ...
    topologyBuilder.setBolt("sum", new SlidingWindowTestSumBolt().withWindow(new BaseWindowedBolt.Count(SUM_BOLT_WINDOW_LENGTH), new BaseWindowedBolt.Count(SUM_BOLT_WINDOW_SLIDE)), 1).shuffleGrouping("spout");
    // the following bolt calculate the average value of elements in the window. The window has length
    // of 3. So it generates the average of 3 elements and then wait for another 3 elements.
    topologyBuilder.setBolt("avg", new SlidingWindowTestAvgBolt().withTumblingWindow(new BaseWindowedBolt.Count(AVG_BOLT_WINDOW_LENGTH)), 1).shuffleGrouping("sum");
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add("SlidingWindowTopologyTest.SpoutAvgSum");
    userDefineMetrics.add("SlidingWindowTopologyTest.BoltAvgSum");
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "SlidingWindowTopologyTest");
    JStormUnitTestValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutAvgSum = (int) metrics.get("SlidingWindowTopologyTest.SpoutAvgSum").doubleValue();
            int boltAvgSum = (int) metrics.get("SlidingWindowTopologyTest.BoltAvgSum").doubleValue();
            System.out.println(spoutAvgSum + " " + boltAvgSum);
            assertEquals(spoutAvgSum, boltAvgSum);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}
Also used : JStormUnitTestValidator(com.jstorm.example.unittests.utils.JStormUnitTestValidator) TopologyBuilder(backtype.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) JStormUnitTestMetricValidator(com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator) Map(java.util.Map) HashMap(java.util.HashMap) BaseWindowedBolt(backtype.storm.topology.base.BaseWindowedBolt) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 88 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class TestSuite method testUserDefine.

@Test
public void testUserDefine() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("user-define-spout", new TestIndexSpout());
    TestIndexBolt esIndexBolt = new TestIndexBolt(esConfig);
    builder.setBolt("user-define-bolt", esIndexBolt).shuffleGrouping("user-define-spout");
    cluster.submitTopology("UserDefine-Test", conf, builder.createTopology());
}
Also used : TestIndexSpout(com.alibaba.jstorm.elasticsearch.index.TestIndexSpout) TopologyBuilder(backtype.storm.topology.TopologyBuilder) TestIndexBolt(com.alibaba.jstorm.elasticsearch.userdefine.TestIndexBolt) Test(org.junit.Test)

Example 89 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class TestSuite method testQuery.

@Test
public void testQuery() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("query-spout", new TestQuerySpout());
    EsOutputDeclarer esOutputDeclarer = new EsOutputDeclarer().addField("date");
    EsQueryBolt esIndexBolt = new EsQueryBolt(esConfig, new TestQueryMapper(), esOutputDeclarer);
    builder.setBolt("query-bolt", esIndexBolt).shuffleGrouping("query-spout");
    builder.setBolt("end-bolt", new TestQueryBolt()).shuffleGrouping("query-bolt");
    cluster.submitTopology("Query-Test", conf, builder.createTopology());
}
Also used : TestQueryBolt(com.alibaba.jstorm.elasticsearch.query.TestQueryBolt) TopologyBuilder(backtype.storm.topology.TopologyBuilder) EsQueryBolt(com.alibaba.jstorm.elasticsearch.bolt.EsQueryBolt) TestQueryMapper(com.alibaba.jstorm.elasticsearch.query.TestQueryMapper) TestQuerySpout(com.alibaba.jstorm.elasticsearch.query.TestQuerySpout) EsOutputDeclarer(com.alibaba.jstorm.elasticsearch.common.EsOutputDeclarer) Test(org.junit.Test)

Example 90 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project storm-hbase by ypf412.

the class HBaseSpoutTest method testHBaseSpout.

@Test
public void testHBaseSpout() throws Exception {
    hbaseUtil.cleanupTestDir();
    HBaseTestUtil.writeLocalHBaseXml(hbaseUtil);
    PropConfig hbasePropConfig;
    try {
        hbasePropConfig = new PropConfig("hbase.properties");
    } catch (IOException e1) {
        throw new RuntimeException(e1);
    }
    String tableName = Constants.HBASE_DEFAULT_TABLE_NAME;
    String tableNameStr = hbasePropConfig.getProperty("hbase.table.name");
    if (tableNameStr != null && !tableNameStr.equals(""))
        tableName = tableNameStr;
    String columnFamily = Constants.HBASE_DEFAULT_COLUMN_FAMILY;
    String columnFamilyStr = hbasePropConfig.getProperty("hbase.table.column_family");
    if (columnFamilyStr != null && !columnFamilyStr.equals(""))
        columnFamily = columnFamilyStr;
    HTable htable = hbaseUtil.createTable(Bytes.toBytes(tableName), Bytes.toBytes(columnFamily));
    HBaseTestUtil.loadStreamDataToHBase(ClassLoader.getSystemResource("datasource.txt").getPath(), htable, hbasePropConfig);
    int count = hbaseUtil.countRows(htable);
    assertTrue(count > 0);
    System.out.println("*** load " + count + " rows into hbase test table: " + tableName);
    stormUtil.getConfig().put(Constants.STORM_PROP_CONF_FILE, "storm.properties");
    stormUtil.getConfig().put(Constants.HBASE_PROP_CONF_FILE, "hbase.properties");
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("hbaseSpout", new HBaseSpout());
    StormTestBolt sinkBolt = new StormTestBolt();
    List<Object[]> tuples = StormTestUtil.loadTuples("datasource.txt");
    for (Object[] tuple : tuples) {
        sinkBolt.expectSeq(tuple);
    }
    builder.setBolt("sinkBolt", sinkBolt).fieldsGrouping("hbaseSpout", new Fields("sharding"));
    stormUtil.submitTopology(builder, 5000);
    hbaseUtil.deleteTable(Bytes.toBytes(tableName));
}
Also used : StormTestBolt(ypf412.storm.StormTestUtil.StormTestBolt) TopologyBuilder(backtype.storm.topology.TopologyBuilder) PropConfig(ypf412.storm.util.PropConfig) IOException(java.io.IOException) HTable(org.apache.hadoop.hbase.client.HTable) HBaseSpout(ypf412.storm.spout.HBaseSpout) Fields(backtype.storm.tuple.Fields) Test(org.junit.Test)

Aggregations

TopologyBuilder (backtype.storm.topology.TopologyBuilder)92 Config (backtype.storm.Config)47 Fields (backtype.storm.tuple.Fields)41 LocalCluster (backtype.storm.LocalCluster)23 JStormHelper (com.alibaba.starter.utils.JStormHelper)16 Map (java.util.Map)15 Test (org.junit.Test)15 HashMap (java.util.HashMap)12 LocalDRPC (backtype.storm.LocalDRPC)7 BoltDeclarer (backtype.storm.topology.BoltDeclarer)6 JStormUnitTestMetricValidator (com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator)5 ArrayList (java.util.ArrayList)5 CoordinatedBolt (backtype.storm.coordination.CoordinatedBolt)4 IdStreamSpec (backtype.storm.coordination.CoordinatedBolt.IdStreamSpec)4 SourceArgs (backtype.storm.coordination.CoordinatedBolt.SourceArgs)4 TestWordSpout (backtype.storm.testing.TestWordSpout)4 HashSet (java.util.HashSet)4 StormTopology (backtype.storm.generated.StormTopology)3 BaseWindowedBolt (backtype.storm.topology.base.BaseWindowedBolt)3 JStormUnitTestValidator (com.jstorm.example.unittests.utils.JStormUnitTestValidator)3