Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 21 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class SequenceTopologyTool method buildTopology.

public StormTopology buildTopology() {
    Config conf = getConf();
    TopologyBuilder builder = new TopologyBuilder();
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    if (!isEnableSplit) {
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    } else {
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint).localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint).shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint).fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID")).fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint).noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        Config.registerSerialization(conf, TradeCustomer.class);
        Config.registerSerialization(conf, Pair.class);
    }
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    return builder.createTopology();
}

Example 22 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class SplitRecord method declareOutputFields.

public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declareStream(SequenceTopologyDef.TRADE_STREAM_ID, new Fields("ID", "TRADE"));
    declarer.declareStream(SequenceTopologyDef.CUSTOMER_STREAM_ID, new Fields("ID", "CUSTOMER"));
}

Example 23 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class FastWordCountSessionProcessingTimeWindowTopology method test.

public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L)).withWindowStateMerger(wordCountBolt), count_Parallelism_hint).fieldsGrouping("spout", new Fields("word"));
    // .allGrouping("spout", Common.WATERMARK_STREAM_ID);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 24 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class FastWordCountTopNWindowTopology method test.

public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    int topN = 10;
    Time win = Time.seconds(10L);
    builder.setBolt("count", new WordCount(topN).timeWindow(win).withStateSize(Time.seconds(120L)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
    builder.setBolt("merge", new MergeTopN(topN).timeWindow(win), 1).allGrouping("count");
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 25 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TransactionFastWordCountWindowedState method test.

public static void test() throws Exception {
    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isLocal) {
        conf.put("tuple.num.per.batch", 100);
        conf.put("transaction.scheduler.spout", false);
        conf.put("transaction.exactly.cache.type", "default");
        conf.put("transaction.topology", true);
    }
    int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);
    builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint);
    builder.setBolt("split", new TxSplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout");
    WordCount wordCount = new WordCount();
    builder.setBolt("count", wordCount.timeWindow(Time.seconds(60L)).withTransactionStateOperator(wordCount), countParallelismHint).fieldsGrouping("split", new Fields("word"));
    builder.enableHdfs();
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}