Examples with SlidingWindowAssigner org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner used on opensource projects

Example 1 with SlidingWindowAssigner

use of org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner in project flink by apache.

the class StreamArrowPythonGroupWindowAggregateFunctionOperatorTest method getTestOperator.

@Override
public AbstractArrowPythonAggregateFunctionOperator getTestOperator(Configuration config, PythonFunctionInfo[] pandasAggregateFunctions, RowType inputType, RowType outputType, int[] groupingSet, int[] udafInputOffsets) {
    long size = 10000L;
    long slide = 5000L;
    SlidingWindowAssigner windowAssigner = SlidingWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(slide)).withEventTime();
    EventTimeTriggers.AfterEndOfWindow<Window> trigger = EventTimeTriggers.afterEndOfWindow();
    RowType udfInputType = (RowType) Projection.of(udafInputOffsets).project(inputType);
    RowType udfOutputType = (RowType) Projection.range(groupingSet.length, outputType.getFieldCount() - 2).project(outputType);
    return new PassThroughStreamArrowPythonGroupWindowAggregateFunctionOperator(config, pandasAggregateFunctions, inputType, udfInputType, udfOutputType, 3, windowAssigner, trigger, 0, new NamedWindowProperty[] { new NamedWindowProperty("start", new WindowStart(null)), new NamedWindowProperty("end", new WindowEnd(null)) }, UTC_ZONE_ID, ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(new TableConfig()), "UdafInputProjection", inputType, udfInputType, udafInputOffsets));
}

Example 2 with SlidingWindowAssigner

use of org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner in project flink by apache.

the class PythonStreamGroupWindowAggregateOperatorTest method getTestOperator.

@Override
OneInputStreamOperator getTestOperator(Configuration config) {
    long size = 10000L;
    long slide = 5000L;
    SlidingWindowAssigner windowAssigner = SlidingWindowAssigner.of(Duration.ofMillis(size), Duration.ofMillis(slide)).withEventTime();
    WindowReference windowRef = new WindowReference("w$", new TimestampType(3));
    return new PassThroughPythonStreamGroupWindowAggregateOperator(config, getInputType(), getOutputType(), new PythonAggregateFunctionInfo[] { new PythonAggregateFunctionInfo(PythonScalarFunctionOperatorTestBase.DummyPythonFunction.INSTANCE, new Integer[] { 2 }, -1, false) }, getGrouping(), -1, false, false, 3, windowAssigner, FlinkFnApi.GroupWindow.WindowType.SLIDING_GROUP_WINDOW, true, true, size, slide, 0L, 0L, new NamedWindowProperty[] { new NamedWindowProperty("start", new WindowStart(null)), new NamedWindowProperty("end", new WindowEnd(null)) }, UTC_ZONE_ID);
}

Example 3 with SlidingWindowAssigner

use of org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner in project flink by apache.

the class WindowTableFunctionUtil method createWindowAssigner.

/**
 * Creates window assigner based on input window strategy.
 *
 * @param windowingStrategy input window strategy
 * @return new created window assigner
 */
public static WindowAssigner<TimeWindow> createWindowAssigner(TimeAttributeWindowingStrategy windowingStrategy) {
    WindowSpec windowSpec = windowingStrategy.getWindow();
    boolean isProctime = windowingStrategy.isProctime();
    if (windowSpec instanceof TumblingWindowSpec) {
        TumblingWindowSpec tumblingWindowSpec = (TumblingWindowSpec) windowSpec;
        TumblingWindowAssigner windowAssigner = TumblingWindowAssigner.of(tumblingWindowSpec.getSize());
        if (isProctime) {
            windowAssigner = windowAssigner.withProcessingTime();
        }
        if (tumblingWindowSpec.getOffset() != null) {
            windowAssigner = windowAssigner.withOffset(tumblingWindowSpec.getOffset());
        }
        return windowAssigner;
    } else if (windowSpec instanceof HoppingWindowSpec) {
        HoppingWindowSpec hoppingWindowSpec = (HoppingWindowSpec) windowSpec;
        SlidingWindowAssigner windowAssigner = SlidingWindowAssigner.of(hoppingWindowSpec.getSize(), hoppingWindowSpec.getSlide());
        if (isProctime) {
            windowAssigner = windowAssigner.withProcessingTime();
        }
        if (hoppingWindowSpec.getOffset() != null) {
            windowAssigner = windowAssigner.withOffset(hoppingWindowSpec.getOffset());
        }
        return windowAssigner;
    } else if (windowSpec instanceof CumulativeWindowSpec) {
        CumulativeWindowSpec cumulativeWindowSpec = (CumulativeWindowSpec) windowSpec;
        CumulativeWindowAssigner windowAssigner = CumulativeWindowAssigner.of(cumulativeWindowSpec.getMaxSize(), cumulativeWindowSpec.getStep());
        if (isProctime) {
            windowAssigner = windowAssigner.withProcessingTime();
        }
        if (cumulativeWindowSpec.getOffset() != null) {
            windowAssigner = windowAssigner.withOffset(cumulativeWindowSpec.getOffset());
        }
        return windowAssigner;
    } else {
        throw new TableException(String.format("Unknown window spec: %s", windowSpec.getClass().getSimpleName()));
    }
}

Example 4 with SlidingWindowAssigner

use of org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner in project flink by apache.

the class WindowTableFunctionOperatorTest method testHopWindows.

@Test
public void testHopWindows() throws Exception {
    final SlidingWindowAssigner assigner = SlidingWindowAssigner.of(Duration.ofSeconds(3), Duration.ofSeconds(1));
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(assigner, shiftTimeZone);
    testHarness.setup(OUT_SERIALIZER);
    testHarness.open();
    // process elements
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.processElement(insertRecord("key1", 1, 20L));
    testHarness.processElement(insertRecord("key2", 1, 3999L));
    testHarness.processWatermark(new Watermark(999));
    // append 3 fields: window_start, window_end, window_time
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(-2000L), localMills(1000L), 999L));
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(-1000L), localMills(2000L), 1999L));
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(0L), localMills(3000L), 2999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(1000L), localMills(4000L), 3999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(2000L), localMills(5000L), 4999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(3000L), localMills(6000L), 5999L));
    expectedOutput.add(new Watermark(999));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // late element would not be dropped
    testHarness.processElement(insertRecord("key2", 1, 80L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(-2000L), localMills(1000L), 999L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(-1000L), localMills(2000L), 1999L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(0L), localMills(3000L), 2999L));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 5 with SlidingWindowAssigner

use of org.apache.flink.table.runtime.operators.window.assigners.SlidingWindowAssigner in project flink by apache.

the class WindowTableFunctionOperatorTest method testProcessingTimeHopWindows.

@Test
public void testProcessingTimeHopWindows() throws Exception {
    final SlidingWindowAssigner assigner = SlidingWindowAssigner.of(Duration.ofSeconds(3), Duration.ofSeconds(1)).withProcessingTime();
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(assigner, shiftTimeZone);
    testHarness.setup(OUT_SERIALIZER);
    testHarness.open();
    // process elements
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    // timestamp is ignored in processing time
    testHarness.setProcessingTime(20L);
    testHarness.processElement(insertRecord("key1", 1, Long.MAX_VALUE));
    testHarness.setProcessingTime(3999L);
    testHarness.processElement(insertRecord("key2", 1, Long.MAX_VALUE));
    // append 3 fields: window_start, window_end, window_time
    expectedOutput.add(insertRecord("key1", 1, Long.MAX_VALUE, localMills(-2000L), localMills(1000L), 999L));
    expectedOutput.add(insertRecord("key1", 1, Long.MAX_VALUE, localMills(-1000L), localMills(2000L), 1999L));
    expectedOutput.add(insertRecord("key1", 1, Long.MAX_VALUE, localMills(0L), localMills(3000L), 2999L));
    expectedOutput.add(insertRecord("key2", 1, Long.MAX_VALUE, localMills(1000L), localMills(4000L), 3999L));
    expectedOutput.add(insertRecord("key2", 1, Long.MAX_VALUE, localMills(2000L), localMills(5000L), 4999L));
    expectedOutput.add(insertRecord("key2", 1, Long.MAX_VALUE, localMills(3000L), localMills(6000L), 5999L));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
}