Search in sources :

Example 1 with Duration

use of org.apache.storm.topology.base.BaseWindowedBolt.Duration in project storm by apache.

the class SlidingTupleTsTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingWindowSumBolt().withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)).withTimestampField("ts").withLag(new Duration(5, TimeUnit.SECONDS));
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology())) {
            Utils.sleep(40000);
        }
    }
}
Also used : LocalCluster(org.apache.storm.LocalCluster) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) RandomIntegerSpout(org.apache.storm.starter.spout.RandomIntegerSpout) Duration(org.apache.storm.topology.base.BaseWindowedBolt.Duration) SlidingWindowSumBolt(org.apache.storm.starter.bolt.SlidingWindowSumBolt) PrinterBolt(org.apache.storm.starter.bolt.PrinterBolt) LocalTopology(org.apache.storm.LocalCluster.LocalTopology) BaseWindowedBolt(org.apache.storm.topology.base.BaseWindowedBolt)

Example 2 with Duration

use of org.apache.storm.topology.base.BaseWindowedBolt.Duration in project storm by apache.

the class JoinExample method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // a stream of (number, square) pairs
    PairStream<Integer, Integer> squares = builder.newStream(new NumberSpout(x -> x * x), new PairValueMapper<>(0, 1));
    // a stream of (number, cube) pairs
    PairStream<Integer, Integer> cubes = builder.newStream(new NumberSpout(x -> x * x * x), new PairValueMapper<>(0, 1));
    // create a windowed stream of five seconds duration
    squares.window(TumblingWindows.of(Duration.seconds(5))).join(cubes).print();
    Config config = new Config();
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}
Also used : StormSubmitter(org.apache.storm.StormSubmitter) OutputFieldsDeclarer(org.apache.storm.topology.OutputFieldsDeclarer) Duration(org.apache.storm.topology.base.BaseWindowedBolt.Duration) BaseRichSpout(org.apache.storm.topology.base.BaseRichSpout) PairValueMapper(org.apache.storm.streams.operations.mappers.PairValueMapper) PairStream(org.apache.storm.streams.PairStream) StreamBuilder(org.apache.storm.streams.StreamBuilder) TopologyContext(org.apache.storm.task.TopologyContext) Fields(org.apache.storm.tuple.Fields) Utils(org.apache.storm.utils.Utils) Function(org.apache.storm.streams.operations.Function) LocalCluster(org.apache.storm.LocalCluster) Values(org.apache.storm.tuple.Values) TumblingWindows(org.apache.storm.streams.windowing.TumblingWindows) Map(java.util.Map) Config(org.apache.storm.Config) SpoutOutputCollector(org.apache.storm.spout.SpoutOutputCollector) LocalCluster(org.apache.storm.LocalCluster) Config(org.apache.storm.Config) StreamBuilder(org.apache.storm.streams.StreamBuilder)

Example 3 with Duration

use of org.apache.storm.topology.base.BaseWindowedBolt.Duration in project storm by apache.

the class WindowedBoltExecutor method initWindowManager.

private WindowManager<Tuple> initWindowManager(WindowLifecycleListener<Tuple> lifecycleListener, Map stormConf, TopologyContext context) {
    WindowManager<Tuple> manager = new WindowManager<>(lifecycleListener);
    Duration windowLengthDuration = null;
    Count windowLengthCount = null;
    Duration slidingIntervalDuration = null;
    Count slidingIntervalCount = null;
    // window length
    if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)) {
        windowLengthCount = new Count(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue());
    } else if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) {
        windowLengthDuration = new Duration(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS);
    }
    // sliding interval
    if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)) {
        slidingIntervalCount = new Count(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue());
    } else if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) {
        slidingIntervalDuration = new Duration(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS);
    } else {
        // default is a sliding window of count 1
        slidingIntervalCount = new Count(1);
    }
    // tuple ts
    if (timestampExtractor != null) {
        // late tuple stream
        lateTupleStream = (String) stormConf.get(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM);
        if (lateTupleStream != null) {
            if (!context.getThisStreams().contains(lateTupleStream)) {
                throw new IllegalArgumentException("Stream for late tuples must be defined with the builder method withLateTupleStream");
            }
        }
        // max lag
        if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) {
            maxLagMs = ((Number) stormConf.get(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)).intValue();
        } else {
            maxLagMs = DEFAULT_MAX_LAG_MS;
        }
        // watermark interval
        int watermarkInterval;
        if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)) {
            watermarkInterval = ((Number) stormConf.get(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue();
        } else {
            watermarkInterval = DEFAULT_WATERMARK_EVENT_INTERVAL_MS;
        }
        waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkInterval, maxLagMs, getComponentStreams(context));
    } else {
        if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) {
            throw new IllegalArgumentException("Late tuple stream can be defined only when specifying a timestamp field");
        }
    }
    // validate
    validate(stormConf, windowLengthCount, windowLengthDuration, slidingIntervalCount, slidingIntervalDuration);
    evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration, manager);
    triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy);
    manager.setEvictionPolicy(evictionPolicy);
    manager.setTriggerPolicy(triggerPolicy);
    return manager;
}
Also used : Duration(org.apache.storm.topology.base.BaseWindowedBolt.Duration) Count(org.apache.storm.topology.base.BaseWindowedBolt.Count) Tuple(org.apache.storm.tuple.Tuple) WindowManager(org.apache.storm.windowing.WindowManager)

Aggregations

Duration (org.apache.storm.topology.base.BaseWindowedBolt.Duration)3 Config (org.apache.storm.Config)2 LocalCluster (org.apache.storm.LocalCluster)2 Map (java.util.Map)1 LocalTopology (org.apache.storm.LocalCluster.LocalTopology)1 StormSubmitter (org.apache.storm.StormSubmitter)1 SpoutOutputCollector (org.apache.storm.spout.SpoutOutputCollector)1 PrinterBolt (org.apache.storm.starter.bolt.PrinterBolt)1 SlidingWindowSumBolt (org.apache.storm.starter.bolt.SlidingWindowSumBolt)1 RandomIntegerSpout (org.apache.storm.starter.spout.RandomIntegerSpout)1 PairStream (org.apache.storm.streams.PairStream)1 StreamBuilder (org.apache.storm.streams.StreamBuilder)1 Function (org.apache.storm.streams.operations.Function)1 PairValueMapper (org.apache.storm.streams.operations.mappers.PairValueMapper)1 TumblingWindows (org.apache.storm.streams.windowing.TumblingWindows)1 TopologyContext (org.apache.storm.task.TopologyContext)1 OutputFieldsDeclarer (org.apache.storm.topology.OutputFieldsDeclarer)1 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)1 BaseRichSpout (org.apache.storm.topology.base.BaseRichSpout)1 BaseWindowedBolt (org.apache.storm.topology.base.BaseWindowedBolt)1