Examples with WindowManager com.twitter.heron.api.windowing.WindowManager used on opensource projects

Example 1 with WindowManager

use of com.twitter.heron.api.windowing.WindowManager in project incubator-heron by apache.

the class WindowedBoltExecutor method initWindowManager.

@SuppressWarnings("unchecked")
private WindowManager<Tuple> initWindowManager(WindowLifecycleListener<Tuple> lifecycleListener, Map<String, Object> topoConf, TopologyContext context, Collection<Event<Tuple>> queue) {
    WindowManager<Tuple> manager = new WindowManager<>(lifecycleListener, queue);
    Count windowLengthCount = null;
    Long slidingIntervalDurationMs = null;
    Count slidingIntervalCount = null;
    // window length
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)) {
        windowLengthCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) {
        windowLengthDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS);
    }
    // sliding interval
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)) {
        slidingIntervalCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) {
        slidingIntervalDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS);
    } else {
        // default is a sliding window of count 1
        slidingIntervalCount = new Count(1);
    }
    // tuple ts
    if (timestampExtractor != null) {
        // late tuple stream
        lateTupleStream = (String) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM);
        if (lateTupleStream != null) {
            if (!context.getThisStreams().contains(lateTupleStream)) {
                throw new IllegalArgumentException("Stream for late tuples must be defined with the " + "builder method withLateTupleStream");
            }
        }
        // max lag
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) {
            maxLagMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)).intValue();
        } else {
            maxLagMs = DEFAULT_MAX_LAG_MS;
        }
        // watermark interval
        long watermarkIntervalMs;
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)) {
            watermarkIntervalMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue();
        } else {
            watermarkIntervalMs = DEFAULT_WATERMARK_EVENT_INTERVAL_MS;
        }
        waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkIntervalMs, maxLagMs, getComponentStreams(context), topoConf);
    } else {
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) {
            throw new IllegalArgumentException("Late tuple stream can be defined only when " + "specifying" + " a timestamp field");
        }
    }
    boolean hasCustomTrigger = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
    boolean hasCustomEvictor = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    if (hasCustomTrigger && hasCustomEvictor) {
        triggerPolicy = (TriggerPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
        evictionPolicy = (EvictionPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    } else if (!hasCustomEvictor && !hasCustomTrigger) {
        // validate
        validate(topoConf, windowLengthCount, windowLengthDurationMs, slidingIntervalCount, slidingIntervalDurationMs);
        evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDurationMs);
        triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDurationMs);
    } else {
        throw new IllegalArgumentException("If either a custom TriggerPolicy or EvictionPolicy is defined, both must be.");
    }
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(topoConf);
    triggerPolicy.setTriggerHandler(manager);
    triggerPolicy.setWindowManager(manager);
    manager.setEvictionPolicy(evictionPolicy);
    manager.setTriggerPolicy(triggerPolicy);
    // restore state if there is existing state
    if (this.state != null && this.state.get(WINDOWING_INTERNAL_STATE) != null && !((HashMapState) this.state.get(WINDOWING_INTERNAL_STATE)).isEmpty()) {
        manager.restoreState((Map<String, Serializable>) state.get(WINDOWING_INTERNAL_STATE));
    }
    return manager;
}