Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 41 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class BatchAckerTest method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);
    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");
    ;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 42 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TransactionTestTopology method test.

public static void test() throws Exception {
    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isLocal) {
        conf.put("tuple.num.per.batch", 5);
        conf.put("transaction.scheduler.spout", false);
        conf.put("transaction.exactly.cache.type", "default");
    }
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isScheduleSpout = JStormUtils.parseBoolean(conf.get("transaction.scheduler.spout"), true);
    if (isScheduleSpout)
        // Generate batch by configured time. "transaction.schedule.batch.delay.ms: 1000 # 1sec"
        builder.setSpout("spout", new ScheduleTxSpout(), spoutParallelism);
    else
        // Generate batch by user when calling emitBarrier
        builder.setSpout("spout", new BasicTxSpout(), spoutParallelism, false);
    builder.setBolt("split", new TxSplitSentence(), splitParallelism).localOrShuffleGrouping("spout");
    builder.setBolt("count", new TxWordCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 43 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentMinMaxOfDevicesTopology method buildDevicesTopology.

/**
 * Creates a topology with device-id and count (which are whole numbers) as
 * tuple fields in a stream and it finally generates result stream based on
 * min amd max with device-id and count values.
 */
public static StormTopology buildDevicesTopology() {
    String deviceID = "device-id";
    String count = "count";
    Fields allFields = new Fields(deviceID, count);
    RandomNumberGeneratorSpout spout = new RandomNumberGeneratorSpout(allFields, 10, 1000);
    TridentTopology topology = new TridentTopology();
    Stream devicesStream = topology.newStream("devicegen-spout", spout).each(allFields, new Debug("##### devices"));
    devicesStream.minBy(deviceID).each(allFields, new Debug("#### device with min id"));
    devicesStream.maxBy(count).each(allFields, new Debug("#### device with max count"));
    return topology.build();
}

Example 44 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentReach method test.

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    conf.setMaxSpoutPending(20);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    try {
        JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 45 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentReach method buildTopology.

public static StormTopology buildTopology(LocalDRPC drpc) {
    TridentTopology topology = new TridentTopology();
    TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
    TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));
    topology.newDRPCStream("reach", drpc).stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"), new ExpandList(), new Fields("follower")).groupBy(new Fields("follower")).aggregate(new One(), new Fields("one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    return topology.build();
}