Examples with TimeWindow org.apache.flink.table.runtime.operators.window.TimeWindow used on opensource projects

Example 11 with TimeWindow

use of org.apache.flink.table.runtime.operators.window.TimeWindow in project flink by apache.

the class HeapWindowsGroupingTest method processTriggerWindow.

private void processTriggerWindow(List<List<Long>> actual, List<TimeWindow> windows, HeapWindowsGrouping grouping) {
    while (grouping.hasTriggerWindow()) {
        RowIterator<BinaryRowData> iter = grouping.buildTriggerWindowElementsIterator();
        TimeWindow window = grouping.getTriggerWindow();
        List<Long> buffer = new ArrayList<>();
        while (iter.advanceNext()) {
            buffer.add(iter.getRow().getLong(0));
        }
        if (!buffer.isEmpty()) {
            actual.add(buffer);
            windows.add(window);
        }
    }
}

Example 12 with TimeWindow

use of org.apache.flink.table.runtime.operators.window.TimeWindow in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method processPythonElement.

public void processPythonElement(byte[] inputBytes) {
    try {
        RowData input = udfInputTypeSerializer.deserialize(new DataInputDeserializer(inputBytes));
        if (input.getByte(0) == NORMAL_RECORD) {
            // normal data
            RowData inputRow = input.getRow(1, inputType.getFieldCount());
            BinaryRowData key = groupKeyProjection.apply(inputRow).copy();
            Map<TimeWindow, List<RowData>> curKeyWindowAccumulateData = windowAccumulateData.computeIfAbsent(key.getString(0).toString(), k -> new HashMap<>());
            Map<TimeWindow, List<RowData>> curKeyWindowRetractData = windowRetractData.computeIfAbsent(key.getString(0).toString(), k -> new HashMap<>());
            long watermark = input.getLong(3);
            // advance watermark
            mockPythonInternalService.advanceWatermark(watermark);
            // get timestamp
            long timestamp = inputRow.getLong(inputTimeFieldIndex);
            timestamp = TimeWindowUtil.toUtcTimestampMills(timestamp, shiftTimeZone);
            Collection<TimeWindow> elementWindows = windowAssigner.assignWindows(inputRow, timestamp);
            for (TimeWindow window : elementWindows) {
                if (RowDataUtil.isAccumulateMsg(inputRow)) {
                    List<RowData> currentWindowDatas = curKeyWindowAccumulateData.computeIfAbsent(window, k -> new LinkedList<>());
                    currentWindowDatas.add(inputRow);
                } else {
                    List<RowData> currentWindowDatas = curKeyWindowRetractData.computeIfAbsent(window, k -> new LinkedList<>());
                    currentWindowDatas.add(inputRow);
                }
            }
            List<TimeWindow> actualWindows = new ArrayList<>(elementWindows.size());
            for (TimeWindow window : elementWindows) {
                if (!isWindowLate(window)) {
                    actualWindows.add(window);
                }
            }
            for (TimeWindow window : actualWindows) {
                boolean triggerResult = onElement(key, window);
                if (triggerResult) {
                    triggerWindowProcess(key, window);
                }
                // register a clean up timer for the window
                registerCleanupTimer(key, window);
            }
        } else {
            RowData timerData = input.getRow(4, 3);
            long timestamp = input.getLong(2);
            RowData key = timerData.getRow(1, getKeyType().getFieldCount());
            byte[] encodedNamespace = timerData.getBinary(2);
            bais.setBuffer(encodedNamespace, 0, encodedNamespace.length);
            TimeWindow window = windowSerializer.deserialize(baisWrapper);
            if (timestamp == window.maxTimestamp()) {
                triggerWindowProcess(key, window);
            }
            cleanWindowIfNeeded(key, window, timestamp);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 13 with TimeWindow

use of org.apache.flink.table.runtime.operators.window.TimeWindow in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method open.

@Override
public void open() throws Exception {
    super.open();
    windowBaos = new ByteArrayOutputStreamWithPos();
    windowBaosWrapper = new DataOutputViewStreamWrapper(windowBaos);
    reusePythonRowData = new UpdatableRowData(GenericRowData.of(NORMAL_RECORD, null, null), 3);
    reusePythonTimerRowData = new UpdatableRowData(GenericRowData.of(TRIGGER_TIMER, null, null), 3);
    reusePythonTimerData = new UpdatableRowData(GenericRowData.of(0, null, null, null), 4);
    reuseJoinedRow = new JoinedRowData();
    windowAggResult = new JoinedRowData();
    reusePythonTimerRowData.setField(2, reusePythonTimerData);
    windowAccumulateData = new HashMap<>();
    windowRetractData = new HashMap<>();
    mockPythonInternalService = (InternalTimerServiceImpl<K, TimeWindow>) getInternalTimerService("python-window-timers", windowSerializer, this.mockPythonWindowOperator);
    this.groupKeyProjection = createProjection("GroupKey", grouping);
    int inputFieldIndex = (int) aggregateFunction.getInputs()[0];
    this.aggExtracter = input -> {
        GenericRowData aggResult = new GenericRowData(1);
        aggResult.setField(0, input.getLong(inputFieldIndex));
        return aggResult;
    };
    this.windowExtractor = window -> {
        GenericRowData windowProperty = new GenericRowData(namedProperties.length);
        for (int i = 0; i < namedProperties.length; i++) {
            switch(namedProperties[i]) {
                case WINDOW_START:
                    windowProperty.setField(i, getShiftEpochMills(window.getStart()));
                    break;
                case WINDOW_END:
                    windowProperty.setField(i, getShiftEpochMills(window.getEnd()));
                    break;
                case ROW_TIME_ATTRIBUTE:
                    windowProperty.setField(i, getShiftEpochMills(window.getEnd() - 1));
                    break;
                case PROC_TIME_ATTRIBUTE:
                    windowProperty.setField(i, -1L);
            }
        }
        return windowProperty;
    };
}

Example 14 with TimeWindow

use of org.apache.flink.table.runtime.operators.window.TimeWindow in project flink by apache.

the class CommonExecWindowTableFunction method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    WindowAssigner<TimeWindow> windowAssigner = createWindowAssigner(windowingStrategy);
    final ZoneId shiftTimeZone = TimeWindowUtil.getShiftTimeZone(windowingStrategy.getTimeAttributeType(), config.getLocalTimeZone());
    WindowTableFunctionOperator windowTableFunctionOperator = new WindowTableFunctionOperator(windowAssigner, windowingStrategy.getTimeAttributeIndex(), shiftTimeZone);
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(WINDOW_TRANSFORMATION, config), windowTableFunctionOperator, InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism());
}

Example 15 with TimeWindow

use of org.apache.flink.table.runtime.operators.window.TimeWindow in project flink by apache.

the class CumulativeWindowAssigner method assignWindows.

@Override
public Collection<TimeWindow> assignWindows(RowData element, long timestamp) {
    List<TimeWindow> windows = new ArrayList<>();
    long start = getWindowStartWithOffset(timestamp, offset, maxSize);
    long lastEnd = start + maxSize;
    long firstEnd = getWindowStartWithOffset(timestamp, offset, step) + step;
    for (long end = firstEnd; end <= lastEnd; end += step) {
        windows.add(new TimeWindow(start, end));
    }
    return windows;
}