Examples with BoltDeclarer backtype.storm.topology.BoltDeclarer used on opensource projects

Example 6 with BoltDeclarer

use of backtype.storm.topology.BoltDeclarer in project jstorm by alibaba.

the class BatchMetaTopology method SetBuilder.

public static TopologyBuilder SetBuilder() {
    BatchTopologyBuilder batchTopologyBuilder = new BatchTopologyBuilder(topologyName);
    MetaSpoutConfig metaSpoutConfig = getMetaSpoutConfig(conf);
    BoltDeclarer rebalanceDeclarer = batchTopologyBuilder.setBolt(BatchMetaRebalance.BOLT_NAME, new BatchMetaRebalance(), 1);
    IBatchSpout batchSpout = new BatchMetaSpout(metaSpoutConfig);
    int spoutParal = JStormUtils.parseInt(conf.get("topology.spout.parallel"), 1);
    BoltDeclarer spoutDeclarer = batchTopologyBuilder.setSpout(BatchMetaSpout.SPOUT_NAME, batchSpout, spoutParal);
    spoutDeclarer.allGrouping(BatchMetaRebalance.BOLT_NAME, BatchMetaRebalance.REBALANCE_STREAM_ID);
    int boltParallel = JStormUtils.parseInt(conf.get("topology.bolt.parallel"), 1);
    BoltDeclarer transformDeclarer = batchTopologyBuilder.setBolt(TransformBolt.BOLT_NAME, new TransformBolt(), boltParallel);
    transformDeclarer.shuffleGrouping(BatchMetaSpout.SPOUT_NAME);
    BoltDeclarer countDeclarer = batchTopologyBuilder.setBolt(CountBolt.COUNT_BOLT_NAME, new CountBolt(), boltParallel);
    countDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
    BoltDeclarer sumDeclarer = batchTopologyBuilder.setBolt(CountBolt.SUM_BOLT_NAME, new CountBolt(), boltParallel);
    sumDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
    BoltDeclarer dbDeclarer = batchTopologyBuilder.setBolt(DBBolt.BOLT_NAME, new DBBolt(), 1);
    dbDeclarer.shuffleGrouping(CountBolt.COUNT_BOLT_NAME).shuffleGrouping(CountBolt.SUM_BOLT_NAME);
    return batchTopologyBuilder.getTopologyBuilder();
}

Example 7 with BoltDeclarer

use of backtype.storm.topology.BoltDeclarer in project jstorm by alibaba.

the class BatchSubtopologyBuilder method extendTopology.

public void extendTopology(TopologyBuilder builder) {
    BoltDeclarer declarer = builder.setBolt(_masterId, new CoordinatedBolt(_masterBolt.bolt), _masterBolt.parallelism);
    for (InputDeclaration decl : _masterBolt.declarations) {
        decl.declare(declarer);
    }
    for (Map conf : _masterBolt.componentConfs) {
        declarer.addConfigurations(conf);
    }
    for (String id : _bolts.keySet()) {
        Component component = _bolts.get(id);
        Map<String, SourceArgs> coordinatedArgs = new HashMap<String, SourceArgs>();
        for (String c : componentBoltSubscriptions(component)) {
            SourceArgs source;
            if (c.equals(_masterId)) {
                source = SourceArgs.single();
            } else {
                source = SourceArgs.all();
            }
            coordinatedArgs.put(c, source);
        }
        BoltDeclarer input = builder.setBolt(id, new CoordinatedBolt(component.bolt, coordinatedArgs, null), component.parallelism);
        for (Map conf : component.componentConfs) {
            input.addConfigurations(conf);
        }
        for (String c : componentBoltSubscriptions(component)) {
            input.directGrouping(c, Constants.COORDINATED_STREAM_ID);
        }
        for (InputDeclaration d : component.declarations) {
            d.declare(input);
        }
    }
}

Example 8 with BoltDeclarer

use of backtype.storm.topology.BoltDeclarer in project jstorm by alibaba.

the class TransactionTopologyBuilder method setBolt.

public BoltDeclarer setBolt(String id, ITransactionBoltExecutor bolt, Number parallelism_hint) {
    upToDownstreamComponentsMap.put(id, new HashSet<String>());
    validateUnusedId(id);
    IRichBolt boltExecutor;
    boolean isStatefulBolt = false;
    if (bolt instanceof ITransactionStatefulBoltExecutor) {
        isStatefulBolt = true;
        boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt);
    } else {
        boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt);
    }
    initCommon(id, boltExecutor, parallelism_hint);
    _bolts.put(id, boltExecutor);
    BoltDeclarer ret = new TransactionBoltDeclarer(id);
    ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt);
    return ret;
}

Example 9 with BoltDeclarer

use of backtype.storm.topology.BoltDeclarer in project jstorm by alibaba.

the class SimpleBatchTopology method SetBuilder.

public static TopologyBuilder SetBuilder() {
    BatchTopologyBuilder topologyBuilder = new BatchTopologyBuilder(topologyName);
    int spoutParallel = JStormUtils.parseInt(conf.get("topology.spout.parallel"), 1);
    BoltDeclarer boltDeclarer = topologyBuilder.setSpout("Spout", new SimpleSpout(), spoutParallel);
    int boltParallel = JStormUtils.parseInt(conf.get("topology.bolt.parallel"), 2);
    topologyBuilder.setBolt("Bolt", new SimpleBolt(), boltParallel).shuffleGrouping("Spout");
    return topologyBuilder.getTopologyBuilder();
}

Example 10 with BoltDeclarer

use of backtype.storm.topology.BoltDeclarer in project jstorm by alibaba.

the class SequenceTopology method SetBuilder.

public static void SetBuilder(TopologyBuilder builder, Map conf) {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    if (!isEnableSplit) {
        BoltDeclarer boltDeclarer = builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint);
        // localFirstGrouping is only for jstorm
        // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME).allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    } else {
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
        Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    }
    // conf.put(Config.TOPOLOGY_DEBUG, false);
    // conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
    // conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    // conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
    // conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
    // conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
}