Examples with Window org.apache.apex.malhar.lib.window.Window used on opensource projects

Example 1 with Window

use of org.apache.apex.malhar.lib.window.Window in project apex-malhar by apache.

the class KeyedWindowedOperatorImpl method accumulateTuple.

@Override
public void accumulateTuple(Tuple.WindowedTuple<KeyValPair<KeyT, InputValT>> tuple) {
    KeyValPair<KeyT, InputValT> kvData = tuple.getValue();
    KeyT key = kvData.getKey();
    for (Window window : tuple.getWindows()) {
        // process each window
        AccumT accum = dataStorage.get(window, key);
        if (accum == null) {
            accum = accumulation.defaultAccumulatedValue();
        }
        InputValT inputValue = kvData.getValue();
        AccumT newValue = accumulation.accumulate(accum, inputValue);
        if ((earlyTriggerMillis > 0 || lateTriggerMillis > 0 || earlyTriggerCount > 0 || lateTriggerCount > 0) && useUpdatedKeyStorage()) {
            updatedKeyStorage.put(window, key);
        }
        dataStorage.put(window, key, newValue);
    }
}

Example 2 with Window

use of org.apache.apex.malhar.lib.window.Window in project apex-malhar by apache.

the class AbstractWindowedOperator method fireTimeTriggers.

private void fireTimeTriggers() {
    if (earlyTriggerMillis > 0 || lateTriggerMillis > 0) {
        for (Map.Entry<Window, WindowState> entry : windowStateMap.entries()) {
            Window window = entry.getKey();
            WindowState windowState = entry.getValue();
            if (windowState.watermarkArrivalTime == -1) {
                if (earlyTriggerMillis > 0 && windowState.lastTriggerFiredTime + earlyTriggerMillis <= currentDerivedTimestamp) {
                    // fire early time triggers
                    fireTrigger(window, windowState);
                }
            } else {
                if (lateTriggerMillis > 0 && windowState.lastTriggerFiredTime + lateTriggerMillis <= currentDerivedTimestamp) {
                    // fire late time triggers
                    fireTrigger(window, windowState);
                }
            }
        }
    }
}

Example 3 with Window

use of org.apache.apex.malhar.lib.window.Window in project apex-malhar by apache.

the class AbstractWindowedOperator method processWatermarkAtEndWindow.

protected void processWatermarkAtEndWindow() {
    long implicitWatermark = -1;
    if (implicitWatermarkGenerator != null) {
        implicitWatermark = implicitWatermarkGenerator.getWatermarkTuple(currentDerivedTimestamp).getTimestamp();
    }
    if (implicitWatermark > nextWatermark) {
        nextWatermark = implicitWatermark;
    }
    if (nextWatermark > 0 && currentWatermark < nextWatermark) {
        long horizon = nextWatermark - allowedLatenessMillis;
        for (Iterator<Map.Entry<Window, WindowState>> it = windowStateMap.entries().iterator(); it.hasNext(); ) {
            Map.Entry<Window, WindowState> entry = it.next();
            Window window = entry.getKey();
            WindowState windowState = entry.getValue();
            if (window.getBeginTimestamp() + window.getDurationMillis() < nextWatermark) {
                // watermark has not arrived for this window before, marking this window late
                if (windowState.watermarkArrivalTime == -1) {
                    windowState.watermarkArrivalTime = currentDerivedTimestamp;
                    if (triggerAtWatermark) {
                        // fire trigger at watermark if applicable
                        fireTrigger(window, windowState);
                    }
                }
                if (allowedLatenessMillis >= 0 && window.getBeginTimestamp() + window.getDurationMillis() < horizon) {
                    // discard this window because it's too late now
                    it.remove();
                    dataStorage.remove(window);
                    if (retractionStorage != null) {
                        retractionStorage.remove(window);
                    }
                }
            }
        }
        streamingWindowToLatenessHorizon.put(streamingWindowId, horizon);
        controlOutput.emit(new WatermarkImpl(nextWatermark));
        this.currentWatermark = nextWatermark;
    }
}

Example 4 with Window

use of org.apache.apex.malhar.lib.window.Window in project apex-malhar by apache.

the class AbstractWindowedOperator method processWindowState.

protected void processWindowState(Tuple.WindowedTuple<? extends Object> windowedTuple) {
    for (Window window : windowedTuple.getWindows()) {
        WindowState windowState = windowStateMap.get(window);
        windowState.tupleCount++;
        // process any count based triggers
        if (windowState.watermarkArrivalTime == -1) {
            // watermark has not arrived yet, check for early count based trigger
            if (earlyTriggerCount > 0 && (windowState.tupleCount % earlyTriggerCount) == 0) {
                fireTrigger(window, windowState);
            }
        } else {
            // watermark has arrived, check for late count based trigger
            if (lateTriggerCount > 0 && (windowState.tupleCount % lateTriggerCount) == 0) {
                fireTrigger(window, windowState);
            }
        }
    }
}

Example 5 with Window

use of org.apache.apex.malhar.lib.window.Window in project apex-malhar by apache.

the class KeyedWindowedMergeOperatorTestApplication method assignTestWindow.

public static Window.TimeWindow assignTestWindow(long timestamp) {
    long beginTimestamp = timestamp - timestamp % windowDuration;
    Window.TimeWindow window = new Window.TimeWindow(beginTimestamp, windowDuration);
    if (!windowStateMap.containsWindow(window)) {
        windowStateMap.put(window, new WindowState());
    }
    return window;
}