Examples with MergeRecord com.alipay.dw.jstorm.example.sequence.bolt.MergeRecord used on opensource projects

Example 1 with MergeRecord

use of com.alipay.dw.jstorm.example.sequence.bolt.MergeRecord in project jstorm by alibaba.

the class SequenceTopologyTool method buildTopology.

public StormTopology buildTopology() {
    Config conf = getConf();
    TopologyBuilder builder = new TopologyBuilder();
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    if (!isEnableSplit) {
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    } else {
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        Config.registerSerialization(conf, TradeCustomer.class);
        Config.registerSerialization(conf, Pair.class);
    }
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    return builder.createTopology();
}

Example 2 with MergeRecord

use of com.alipay.dw.jstorm.example.sequence.bolt.MergeRecord in project jstorm by alibaba.

the class SequenceTopology method SetBuilder.

public static void SetBuilder(TopologyBuilder builder, Map conf) {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    if (!isEnableSplit) {
        BoltDeclarer boltDeclarer = builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint);
        // localFirstGrouping is only for jstorm
        // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME).allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    } else {
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
        Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    }
    // conf.put(Config.TOPOLOGY_DEBUG, false);
    // conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
    // conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    // conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
    // conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
    // conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
}