Examples with Tuple org.apache.heron.api.tuple.Tuple used on opensource projects

Example 11 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class StatefulWindowSumBolt method execute.

@Override
public void execute(TupleWindow inputWindow) {
    for (Tuple tuple : inputWindow.get()) {
        System.out.println("Adding to sum: " + tuple.getLongByField("value"));
        sum += tuple.getLongByField("value");
        System.out.println("Sum is now: " + sum);
    }
    collector.emit(new Values(sum));
}

Example 12 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class SlidingWindowSumBolt method execute.

@Override
public void execute(TupleWindow inputWindow) {
    /*
             * The inputWindow gives a view of
             * (a) all the events in the window
             * (b) events that expired since last activation of the window
             * (c) events that newly arrived since last activation of the window
             */
    List<Tuple> tuplesInWindow = inputWindow.get();
    List<Tuple> newTuples = inputWindow.getNew();
    List<Tuple> expiredTuples = inputWindow.getExpired();
    LOG.fine("Events in current window: " + tuplesInWindow.size());
    /*
             * Instead of iterating over all the tuples in the window to compute
             * the sum, the values for the new events are added and old events are
             * subtracted. Similar optimizations might be possible in other
             * windowing computations.
             */
    for (Tuple tuple : newTuples) {
        sum += (int) tuple.getValue(0);
    }
    for (Tuple tuple : expiredTuples) {
        sum -= (int) tuple.getValue(0);
    }
    collector.emit(new Values(sum));
}

Example 13 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class WindowedBoltExecutor method initWindowManager.

@SuppressWarnings("unchecked")
private WindowManager<Tuple> initWindowManager(WindowLifecycleListener<Tuple> lifecycleListener, Map<String, Object> topoConf, TopologyContext context, Collection<Event<Tuple>> queue) {
    WindowManager<Tuple> manager = new WindowManager<>(lifecycleListener, queue);
    Count windowLengthCount = null;
    Long slidingIntervalDurationMs = null;
    Count slidingIntervalCount = null;
    // window length
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)) {
        windowLengthCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) {
        windowLengthDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS);
    }
    // sliding interval
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)) {
        slidingIntervalCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) {
        slidingIntervalDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS);
    } else {
        // default is a sliding window of count 1
        slidingIntervalCount = new Count(1);
    }
    // tuple ts
    if (timestampExtractor != null) {
        // late tuple stream
        lateTupleStream = (String) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM);
        if (lateTupleStream != null) {
            if (!context.getThisStreams().contains(lateTupleStream)) {
                throw new IllegalArgumentException("Stream for late tuples must be defined with the " + "builder method withLateTupleStream");
            }
        }
        // max lag
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) {
            maxLagMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)).intValue();
        } else {
            maxLagMs = DEFAULT_MAX_LAG_MS;
        }
        // watermark interval
        long watermarkIntervalMs;
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)) {
            watermarkIntervalMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue();
        } else {
            watermarkIntervalMs = DEFAULT_WATERMARK_EVENT_INTERVAL_MS;
        }
        waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkIntervalMs, maxLagMs, getComponentStreams(context), topoConf);
    } else {
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) {
            throw new IllegalArgumentException("Late tuple stream can be defined only when " + "specifying" + " a timestamp field");
        }
    }
    boolean hasCustomTrigger = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
    boolean hasCustomEvictor = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    if (hasCustomTrigger && hasCustomEvictor) {
        triggerPolicy = (TriggerPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
        evictionPolicy = (EvictionPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    } else if (!hasCustomEvictor && !hasCustomTrigger) {
        // validate
        validate(topoConf, windowLengthCount, windowLengthDurationMs, slidingIntervalCount, slidingIntervalDurationMs);
        evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDurationMs);
        triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDurationMs);
    } else {
        throw new IllegalArgumentException("If either a custom TriggerPolicy or EvictionPolicy is defined, both must be.");
    }
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(topoConf);
    triggerPolicy.setTriggerHandler(manager);
    triggerPolicy.setWindowManager(manager);
    manager.setEvictionPolicy(evictionPolicy);
    manager.setTriggerPolicy(triggerPolicy);
    // restore state if there is existing state
    if (this.state != null && this.state.get(WINDOWING_INTERNAL_STATE) != null && !((HashMapState) this.state.get(WINDOWING_INTERNAL_STATE)).isEmpty()) {
        manager.restoreState((Map<String, Serializable>) state.get(WINDOWING_INTERNAL_STATE));
    }
    return manager;
}

Example 14 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(org.apache.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 15 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class GeneralReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private GeneralReduceByKeyAndWindowOperator<KeyValue<String, Integer>, String, Integer> getReduceByWindowOperator(Integer identity) {
    GeneralReduceByKeyAndWindowOperator<KeyValue<String, Integer>, String, Integer> reduceByWindowOperator = new GeneralReduceByKeyAndWindowOperator<>(x -> x.getKey(), identity, (o, o2) -> o + o2.getValue());
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}