Examples with SlidingGroupWindow org.apache.flink.table.planner.plan.logical.SlidingGroupWindow used on opensource projects

Example 1 with SlidingGroupWindow

use of org.apache.flink.table.planner.plan.logical.SlidingGroupWindow in project flink by apache.

the class StreamExecGroupWindowAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final boolean isCountWindow;
    if (window instanceof TumblingGroupWindow) {
        isCountWindow = hasRowIntervalType(((TumblingGroupWindow) window).size());
    } else if (window instanceof SlidingGroupWindow) {
        isCountWindow = hasRowIntervalType(((SlidingGroupWindow) window).size());
    } else {
        isCountWindow = false;
    }
    if (isCountWindow && grouping.length > 0 && config.getStateRetentionTime() < 0) {
        LOGGER.warn("No state retention interval configured for a query which accumulates state. " + "Please provide a query configuration with valid retention interval to prevent " + "excessive state size. You may specify a retention time of 0 to not clean up the state.");
    }
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    final int inputTimeFieldIndex;
    if (isRowtimeAttribute(window.timeAttribute())) {
        inputTimeFieldIndex = timeFieldIndex(FlinkTypeFactory.INSTANCE().buildRelNodeRowType(inputRowType), planner.getRelBuilder(), window.timeAttribute());
        if (inputTimeFieldIndex < 0) {
            throw new TableException("Group window must defined on a time attribute, " + "but the time attribute can't be found.\n" + "This should never happen. Please file an issue.");
        }
    } else {
        inputTimeFieldIndex = -1;
    }
    final ZoneId shiftTimeZone = TimeWindowUtil.getShiftTimeZone(window.timeAttribute().getOutputDataType().getLogicalType(), config.getLocalTimeZone());
    final boolean[] aggCallNeedRetractions = new boolean[aggCalls.length];
    Arrays.fill(aggCallNeedRetractions, needRetraction);
    final AggregateInfoList aggInfoList = transformToStreamAggregateInfoList(inputRowType, JavaScalaConversionUtil.toScala(Arrays.asList(aggCalls)), aggCallNeedRetractions, needRetraction, // isStateBackendDataViews
    true, // needDistinctInfo
    true);
    final GeneratedClass<?> aggCodeGenerator = createAggsHandler(aggInfoList, config, planner.getRelBuilder(), inputRowType.getChildren(), shiftTimeZone);
    final LogicalType[] aggResultTypes = extractLogicalTypes(aggInfoList.getActualValueTypes());
    final LogicalType[] windowPropertyTypes = Arrays.stream(namedWindowProperties).map(p -> p.getProperty().getResultType()).toArray(LogicalType[]::new);
    final EqualiserCodeGenerator generator = new EqualiserCodeGenerator(ArrayUtils.addAll(aggResultTypes, windowPropertyTypes));
    final GeneratedRecordEqualiser equaliser = generator.generateRecordEqualiser("WindowValueEqualiser");
    final LogicalType[] aggValueTypes = extractLogicalTypes(aggInfoList.getActualValueTypes());
    final LogicalType[] accTypes = extractLogicalTypes(aggInfoList.getAccTypes());
    final int inputCountIndex = aggInfoList.getIndexOfCountStar();
    final WindowOperator<?, ?> operator = createWindowOperator(config, aggCodeGenerator, equaliser, accTypes, windowPropertyTypes, aggValueTypes, inputRowType.getChildren().toArray(new LogicalType[0]), inputTimeFieldIndex, shiftTimeZone, inputCountIndex);
    final OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(GROUP_WINDOW_AGGREGATE_TRANSFORMATION, config), operator, InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism());
    // set KeyType and Selector for state
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(grouping, InternalTypeInfo.of(inputRowType));
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}

Example 2 with SlidingGroupWindow

use of org.apache.flink.table.planner.plan.logical.SlidingGroupWindow in project flink by apache.

the class StreamExecGroupWindowAggregate method createAggsHandler.

private GeneratedClass<?> createAggsHandler(AggregateInfoList aggInfoList, ExecNodeConfig config, RelBuilder relBuilder, List<LogicalType> fieldTypes, ZoneId shiftTimeZone) {
    final boolean needMerge;
    final Class<?> windowClass;
    if (window instanceof SlidingGroupWindow) {
        ValueLiteralExpression size = ((SlidingGroupWindow) window).size();
        needMerge = hasTimeIntervalType(size);
        windowClass = hasRowIntervalType(size) ? CountWindow.class : TimeWindow.class;
    } else if (window instanceof TumblingGroupWindow) {
        needMerge = false;
        ValueLiteralExpression size = ((TumblingGroupWindow) window).size();
        windowClass = hasRowIntervalType(size) ? CountWindow.class : TimeWindow.class;
    } else if (window instanceof SessionGroupWindow) {
        needMerge = true;
        windowClass = TimeWindow.class;
    } else {
        throw new TableException("Unsupported window: " + window.toString());
    }
    final AggsHandlerCodeGenerator generator = new AggsHandlerCodeGenerator(new CodeGeneratorContext(config.getTableConfig()), relBuilder, JavaScalaConversionUtil.toScala(fieldTypes), // copyInputField
    false).needAccumulate();
    if (needMerge) {
        generator.needMerge(0, false, null);
    }
    if (needRetraction) {
        generator.needRetract();
    }
    final List<WindowProperty> windowProperties = Arrays.asList(Arrays.stream(namedWindowProperties).map(NamedWindowProperty::getProperty).toArray(WindowProperty[]::new));
    final boolean isTableAggregate = isTableAggregate(Arrays.asList(aggInfoList.getActualAggregateCalls()));
    if (isTableAggregate) {
        return generator.generateNamespaceTableAggsHandler("GroupingWindowTableAggsHandler", aggInfoList, JavaScalaConversionUtil.toScala(windowProperties), windowClass, shiftTimeZone);
    } else {
        return generator.generateNamespaceAggsHandler("GroupingWindowAggsHandler", aggInfoList, JavaScalaConversionUtil.toScala(windowProperties), windowClass, shiftTimeZone);
    }
}

Example 3 with SlidingGroupWindow

use of org.apache.flink.table.planner.plan.logical.SlidingGroupWindow in project flink by apache.

the class StreamExecPythonGroupWindowAggregate method generateWindowAssignerAndTrigger.

private Tuple2<WindowAssigner<?>, Trigger<?>> generateWindowAssignerAndTrigger() {
    WindowAssigner<?> windowAssiger;
    Trigger<?> trigger;
    if (window instanceof TumblingGroupWindow) {
        TumblingGroupWindow tumblingWindow = (TumblingGroupWindow) window;
        FieldReferenceExpression timeField = tumblingWindow.timeField();
        ValueLiteralExpression size = tumblingWindow.size();
        if (isProctimeAttribute(timeField) && hasTimeIntervalType(size)) {
            windowAssiger = TumblingWindowAssigner.of(toDuration(size)).withProcessingTime();
            trigger = ProcessingTimeTriggers.afterEndOfWindow();
        } else if (isRowtimeAttribute(timeField) && hasTimeIntervalType(size)) {
            windowAssiger = TumblingWindowAssigner.of(toDuration(size)).withEventTime();
            trigger = EventTimeTriggers.afterEndOfWindow();
        } else if (isProctimeAttribute(timeField) && hasRowIntervalType(size)) {
            windowAssiger = CountTumblingWindowAssigner.of(toLong(size));
            trigger = ElementTriggers.count(toLong(size));
        } else {
            // ProcessingTimeTumblingGroupWindow
            throw new UnsupportedOperationException("Event-time grouping windows on row intervals are currently not supported.");
        }
    } else if (window instanceof SlidingGroupWindow) {
        SlidingGroupWindow slidingWindow = (SlidingGroupWindow) window;
        FieldReferenceExpression timeField = slidingWindow.timeField();
        ValueLiteralExpression size = slidingWindow.size();
        ValueLiteralExpression slide = slidingWindow.slide();
        if (isProctimeAttribute(timeField) && hasTimeIntervalType(size)) {
            windowAssiger = SlidingWindowAssigner.of(toDuration(size), toDuration(slide)).withProcessingTime();
            trigger = ProcessingTimeTriggers.afterEndOfWindow();
        } else if (isRowtimeAttribute(timeField) && hasTimeIntervalType(size)) {
            windowAssiger = SlidingWindowAssigner.of(toDuration(size), toDuration(slide));
            trigger = EventTimeTriggers.afterEndOfWindow();
        } else if (isProctimeAttribute(timeField) && hasRowIntervalType(size)) {
            windowAssiger = CountSlidingWindowAssigner.of(toLong(size), toLong(slide));
            trigger = ElementTriggers.count(toLong(size));
        } else {
            // ProcessingTimeTumblingGroupWindow
            throw new UnsupportedOperationException("Event-time grouping windows on row intervals are currently not supported.");
        }
    } else if (window instanceof SessionGroupWindow) {
        SessionGroupWindow sessionWindow = (SessionGroupWindow) window;
        FieldReferenceExpression timeField = sessionWindow.timeField();
        ValueLiteralExpression gap = sessionWindow.gap();
        if (isProctimeAttribute(timeField)) {
            windowAssiger = SessionWindowAssigner.withGap(toDuration(gap));
            trigger = ProcessingTimeTriggers.afterEndOfWindow();
        } else if (isRowtimeAttribute(timeField)) {
            windowAssiger = SessionWindowAssigner.withGap(toDuration(gap));
            trigger = EventTimeTriggers.afterEndOfWindow();
        } else {
            throw new UnsupportedOperationException("This should not happen.");
        }
    } else {
        throw new TableException("Unsupported window: " + window.toString());
    }
    return Tuple2.of(windowAssiger, trigger);
}

Example 4 with SlidingGroupWindow

use of org.apache.flink.table.planner.plan.logical.SlidingGroupWindow in project flink by apache.

the class LogicalWindowJsonDeserializer method deserialize.

@Override
public LogicalWindow deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
    JsonNode jsonNode = jsonParser.readValueAsTree();
    String kind = jsonNode.get(FIELD_NAME_KIND).asText().toUpperCase();
    WindowReference alias = deserializationContext.readValue(jsonNode.get(FIELD_NAME_ALIAS).traverse(jsonParser.getCodec()), WindowReference.class);
    FieldReferenceExpression timeField = deserializeFieldReferenceExpression(jsonNode.get(FIELD_NAME_TIME_FIELD), jsonParser, deserializationContext);
    switch(kind) {
        case KIND_TUMBLING:
            boolean isTimeTumblingWindow = jsonNode.get(FIELD_NAME_IS_TIME_WINDOW).asBoolean();
            if (isTimeTumblingWindow) {
                Duration size = deserializationContext.readValue(traverse(jsonNode.get(FIELD_NAME_SIZE), jsonParser.getCodec()), Duration.class);
                return new TumblingGroupWindow(alias, timeField, new ValueLiteralExpression(size));
            } else {
                long size = jsonNode.get(FIELD_NAME_SIZE).asLong();
                return new TumblingGroupWindow(alias, timeField, new ValueLiteralExpression(size));
            }
        case KIND_SLIDING:
            boolean isTimeSlidingWindow = jsonNode.get(FIELD_NAME_IS_TIME_WINDOW).asBoolean();
            if (isTimeSlidingWindow) {
                Duration size = deserializationContext.readValue(traverse(jsonNode.get(FIELD_NAME_SIZE), jsonParser.getCodec()), Duration.class);
                Duration slide = deserializationContext.readValue(traverse(jsonNode.get(FIELD_NAME_SLIDE), jsonParser.getCodec()), Duration.class);
                return new SlidingGroupWindow(alias, timeField, new ValueLiteralExpression(size), new ValueLiteralExpression(slide));
            } else {
                long size = jsonNode.get(FIELD_NAME_SIZE).asLong();
                long slide = jsonNode.get(FIELD_NAME_SLIDE).asLong();
                return new SlidingGroupWindow(alias, timeField, new ValueLiteralExpression(size), new ValueLiteralExpression(slide));
            }
        case KIND_SESSION:
            Duration gap = deserializationContext.readValue(traverse(jsonNode.get(FIELD_NAME_GAP), jsonParser.getCodec()), Duration.class);
            return new SessionGroupWindow(alias, timeField, new ValueLiteralExpression(gap));
        default:
            throw new TableException("Unknown Logical Window:" + jsonNode);
    }
}

Example 5 with SlidingGroupWindow

use of org.apache.flink.table.planner.plan.logical.SlidingGroupWindow in project flink by apache.

the class LogicalWindowJsonSerializer method serialize.

@Override
public void serialize(LogicalWindow logicalWindow, JsonGenerator gen, SerializerProvider serializerProvider) throws IOException {
    gen.writeStartObject();
    if (logicalWindow instanceof TumblingGroupWindow) {
        TumblingGroupWindow window = (TumblingGroupWindow) logicalWindow;
        gen.writeStringField(FIELD_NAME_KIND, KIND_TUMBLING);
        serializerProvider.defaultSerializeField(FIELD_NAME_ALIAS, window.aliasAttribute(), gen);
        FieldReferenceExpression timeField = logicalWindow.timeAttribute();
        serializeFieldReferenceExpression(timeField, gen, serializerProvider);
        ValueLiteralExpression size = window.size();
        if (hasTimeIntervalType(size)) {
            Duration duration = toDuration(size);
            gen.writeBooleanField(FIELD_NAME_IS_TIME_WINDOW, true);
            gen.writeObjectField(FIELD_NAME_SIZE, duration);
        } else {
            Long duration = toLong(size);
            gen.writeBooleanField(FIELD_NAME_IS_TIME_WINDOW, false);
            gen.writeNumberField(FIELD_NAME_SIZE, duration);
        }
    } else if (logicalWindow instanceof SlidingGroupWindow) {
        SlidingGroupWindow window = (SlidingGroupWindow) logicalWindow;
        gen.writeStringField(FIELD_NAME_KIND, KIND_SLIDING);
        serializerProvider.defaultSerializeField(FIELD_NAME_ALIAS, window.aliasAttribute(), gen);
        serializeFieldReferenceExpression(window.timeAttribute(), gen, serializerProvider);
        ValueLiteralExpression size = window.size();
        if (hasTimeIntervalType(size)) {
            Duration duration = toDuration(size);
            gen.writeBooleanField(FIELD_NAME_IS_TIME_WINDOW, true);
            gen.writeObjectField(FIELD_NAME_SIZE, duration);
            gen.writeObjectField(FIELD_NAME_SLIDE, toDuration(window.slide()));
        } else {
            Long duration = toLong(size);
            gen.writeBooleanField(FIELD_NAME_IS_TIME_WINDOW, false);
            gen.writeObjectField(FIELD_NAME_SIZE, duration);
            gen.writeObjectField(FIELD_NAME_SLIDE, toLong(window.slide()));
        }
    } else if (logicalWindow instanceof SessionGroupWindow) {
        gen.writeStringField(FIELD_NAME_KIND, KIND_SESSION);
        SessionGroupWindow window = (SessionGroupWindow) logicalWindow;
        serializerProvider.defaultSerializeField(FIELD_NAME_ALIAS, window.aliasAttribute(), gen);
        serializeFieldReferenceExpression(window.timeAttribute(), gen, serializerProvider);
        gen.writeObjectField(FIELD_NAME_GAP, toDuration(window.gap()));
    } else {
        throw new TableException("Unknown LogicalWindow: " + logicalWindow);
    }
    gen.writeEndObject();
}