Examples with MetaSpoutConfig com.alibaba.jstorm.batch.meta.MetaSpoutConfig used on opensource projects

Example 1 with MetaSpoutConfig

use of com.alibaba.jstorm.batch.meta.MetaSpoutConfig in project jstorm by alibaba.

the class BatchMetaTopology method getMetaSpoutConfig.

public static MetaSpoutConfig getMetaSpoutConfig(Map conf) {
    String consumerGroup = (String) conf.get("meta.consumer.group");
    String topic = (String) conf.get("meta.topic");
    String nameServer = (String) conf.get("meta.nameserver");
    String subExpress = (String) conf.get("meta.subexpress");
    String startTimestampStr = (String) conf.get("meta.start.timestamp");
    Long startTimeStamp = null;
    if (startTimestampStr != null) {
        Date date = TimeFormat.getSecond(startTimestampStr);
        startTimeStamp = date.getTime();
    }
    int batchMessageNum = JStormUtils.parseInt(conf.get("meta.batch.message.num"), 1024);
    int maxFailTimes = JStormUtils.parseInt(conf.get("meta.max.fail.times"), 10);
    MetaSpoutConfig ret = new MetaSpoutConfig(consumerGroup, nameServer, topic, subExpress);
    ret.setStartTimeStamp(startTimeStamp);
    ret.setBatchMsgNum(batchMessageNum);
    ret.setMaxFailTimes(maxFailTimes);
    return ret;
}

Example 2 with MetaSpoutConfig

use of com.alibaba.jstorm.batch.meta.MetaSpoutConfig in project jstorm by alibaba.

the class BatchMetaTopology method SetBuilder.

public static TopologyBuilder SetBuilder() {
    BatchTopologyBuilder batchTopologyBuilder = new BatchTopologyBuilder(topologyName);
    MetaSpoutConfig metaSpoutConfig = getMetaSpoutConfig(conf);
    BoltDeclarer rebalanceDeclarer = batchTopologyBuilder.setBolt(BatchMetaRebalance.BOLT_NAME, new BatchMetaRebalance(), 1);
    IBatchSpout batchSpout = new BatchMetaSpout(metaSpoutConfig);
    int spoutParal = JStormUtils.parseInt(conf.get("topology.spout.parallel"), 1);
    BoltDeclarer spoutDeclarer = batchTopologyBuilder.setSpout(BatchMetaSpout.SPOUT_NAME, batchSpout, spoutParal);
    spoutDeclarer.allGrouping(BatchMetaRebalance.BOLT_NAME, BatchMetaRebalance.REBALANCE_STREAM_ID);
    int boltParallel = JStormUtils.parseInt(conf.get("topology.bolt.parallel"), 1);
    BoltDeclarer transformDeclarer = batchTopologyBuilder.setBolt(TransformBolt.BOLT_NAME, new TransformBolt(), boltParallel);
    transformDeclarer.shuffleGrouping(BatchMetaSpout.SPOUT_NAME);
    BoltDeclarer countDeclarer = batchTopologyBuilder.setBolt(CountBolt.COUNT_BOLT_NAME, new CountBolt(), boltParallel);
    countDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
    BoltDeclarer sumDeclarer = batchTopologyBuilder.setBolt(CountBolt.SUM_BOLT_NAME, new CountBolt(), boltParallel);
    sumDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
    BoltDeclarer dbDeclarer = batchTopologyBuilder.setBolt(DBBolt.BOLT_NAME, new DBBolt(), 1);
    dbDeclarer.shuffleGrouping(CountBolt.COUNT_BOLT_NAME).shuffleGrouping(CountBolt.SUM_BOLT_NAME);
    return batchTopologyBuilder.getTopologyBuilder();
}