Search in sources :

Example 21 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SequenceTopology method SetRemoteTopology.

public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    TopologyBuilder builder = new TopologyBuilder();
    SetBuilder(builder, conf);
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
}
Also used : TopologyBuilder(backtype.storm.topology.TopologyBuilder)

Example 22 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SequenceTopologyTool method buildTopology.

public StormTopology buildTopology() {
    Config conf = getConf();
    TopologyBuilder builder = new TopologyBuilder();
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    if (!isEnableSplit) {
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    } else {
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        Config.registerSerialization(conf, TradeCustomer.class);
        Config.registerSerialization(conf, Pair.class);
    }
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    return builder.createTopology();
}
Also used : TotalCount(com.alipay.dw.jstorm.example.userdefined.metrics.TotalCount) Fields(backtype.storm.tuple.Fields) SplitRecord(com.alipay.dw.jstorm.example.sequence.bolt.SplitRecord) TopologyBuilder(backtype.storm.topology.TopologyBuilder) Config(backtype.storm.Config) PairCount(com.alipay.dw.jstorm.example.sequence.bolt.PairCount) SequenceSpout(com.alipay.dw.jstorm.example.sequence.spout.SequenceSpout) MergeRecord(com.alipay.dw.jstorm.example.sequence.bolt.MergeRecord)

Example 23 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class FastWordCountSessionProcessingTimeWindowTopology method test.

public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L)).withWindowStateMerger(wordCountBolt), count_Parallelism_hint).fieldsGrouping("spout", new Fields("word"));
    // .allGrouping("spout", Common.WATERMARK_STREAM_ID);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : JStormHelper(com.alibaba.starter.utils.JStormHelper) Fields(backtype.storm.tuple.Fields) TopologyBuilder(backtype.storm.topology.TopologyBuilder)

Example 24 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class FastWordCountTopNWindowTopology method test.

public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    int topN = 10;
    Time win = Time.seconds(10L);
    builder.setBolt("count", new WordCount(topN).timeWindow(win).withStateSize(Time.seconds(120L)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
    builder.setBolt("merge", new MergeTopN(topN).timeWindow(win), 1).allGrouping("count");
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : TopologyBuilder(backtype.storm.topology.TopologyBuilder) Time(com.alibaba.jstorm.window.Time) JStormHelper(com.alibaba.starter.utils.JStormHelper) Fields(backtype.storm.tuple.Fields)

Example 25 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class PerformanceTestTopology method SetRemoteTopology.

public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    TopologyBuilder builder = new TopologyBuilder();
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
}
Also used : TopologyBuilder(backtype.storm.topology.TopologyBuilder) BoltDeclarer(backtype.storm.topology.BoltDeclarer)

Aggregations

TopologyBuilder (backtype.storm.topology.TopologyBuilder)92 Config (backtype.storm.Config)47 Fields (backtype.storm.tuple.Fields)41 LocalCluster (backtype.storm.LocalCluster)23 JStormHelper (com.alibaba.starter.utils.JStormHelper)16 Map (java.util.Map)15 Test (org.junit.Test)15 HashMap (java.util.HashMap)12 LocalDRPC (backtype.storm.LocalDRPC)7 BoltDeclarer (backtype.storm.topology.BoltDeclarer)6 JStormUnitTestMetricValidator (com.jstorm.example.unittests.utils.JStormUnitTestMetricValidator)5 ArrayList (java.util.ArrayList)5 CoordinatedBolt (backtype.storm.coordination.CoordinatedBolt)4 IdStreamSpec (backtype.storm.coordination.CoordinatedBolt.IdStreamSpec)4 SourceArgs (backtype.storm.coordination.CoordinatedBolt.SourceArgs)4 TestWordSpout (backtype.storm.testing.TestWordSpout)4 HashSet (java.util.HashSet)4 StormTopology (backtype.storm.generated.StormTopology)3 BaseWindowedBolt (backtype.storm.topology.base.BaseWindowedBolt)3 JStormUnitTestValidator (com.jstorm.example.unittests.utils.JStormUnitTestValidator)3