Examples with FlinkRelDistribution org.apache.flink.table.planner.plan.trait.FlinkRelDistribution used on opensource projects

Example 1 with FlinkRelDistribution

use of org.apache.flink.table.planner.plan.trait.FlinkRelDistribution in project flink by apache.

the class StreamPhysicalPythonOverAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalOverAggregate logicWindow = (FlinkLogicalOverAggregate) rel;
    if (logicWindow.groups.size() > 1) {
        throw new TableException("Over Agg: Unsupported use of OVER windows. " + "All aggregates must be computed on the same window. " + "please re-check the over window statement.");
    }
    ImmutableBitSet keys = logicWindow.groups.get(0).keys;
    FlinkRelDistribution requiredDistribution;
    if (!keys.isEmpty()) {
        requiredDistribution = FlinkRelDistribution.hash(keys.asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelNode input = logicWindow.getInput();
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    return new StreamPhysicalPythonOverAggregate(rel.getCluster(), providedTraitSet, newInput, rel.getRowType(), logicWindow);
}

Example 2 with FlinkRelDistribution

use of org.apache.flink.table.planner.plan.trait.FlinkRelDistribution in project flink by apache.

the class StreamPhysicalPythonGroupAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalAggregate agg = (FlinkLogicalAggregate) rel;
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = rel.getCluster().getPlanner().emptyTraitSet().replace(requiredDistribution).replace(FlinkConventions.STREAM_PHYSICAL());
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(agg.getInput(), requiredTraitSet);
    return new StreamPhysicalPythonGroupAggregate(rel.getCluster(), providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(agg.getAggCallList()));
}

Example 3 with FlinkRelDistribution

use of org.apache.flink.table.planner.plan.trait.FlinkRelDistribution in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalWindowAggregate agg = (FlinkLogicalWindowAggregate) rel;
    LogicalWindow window = agg.getWindow();
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean isPandasPythonUDAF = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    if (isPandasPythonUDAF && window instanceof SessionGroupWindow) {
        throw new TableException("Session Group Window is currently not supported for Pandas UDAF.");
    }
    RelNode input = agg.getInput();
    RelOptCluster cluster = rel.getCluster();
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    ReadableConfig config = ShortcutUtils.unwrapTableConfig(rel);
    WindowEmitStrategy emitStrategy = WindowEmitStrategy.apply(config, agg.getWindow());
    if (emitStrategy.produceUpdates()) {
        throw new TableException("Python Group Window Aggregate Function is currently not supported for early fired or lately fired.");
    }
    return new StreamPhysicalPythonGroupWindowAggregate(cluster, providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(aggCalls), agg.getWindow(), agg.getNamedProperties(), emitStrategy);
}

Example 4 with FlinkRelDistribution

use of org.apache.flink.table.planner.plan.trait.FlinkRelDistribution in project flink by apache.

the class TwoStageOptimizedWindowAggregateRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWindowAggregate windowAgg = call.rel(0);
    final RelNode realInput = call.rel(2);
    final WindowingStrategy windowing = windowAgg.windowing();
    RelTraitSet localTraitSet = realInput.getTraitSet().plus(ModifyKindSetTrait.INSERT_ONLY()).plus(UpdateKindTrait.NONE());
    StreamPhysicalLocalWindowAggregate localAgg = new StreamPhysicalLocalWindowAggregate(windowAgg.getCluster(), localTraitSet, realInput, windowAgg.grouping(), windowAgg.aggCalls(), windowing);
    // grouping keys is forwarded by local agg, use indices instead of groupings
    int[] globalGrouping = IntStream.range(0, windowAgg.grouping().length).toArray();
    FlinkRelDistribution globalDistribution = createDistribution(globalGrouping);
    // create exchange if needed
    RelNode newInput = FlinkExpandConversionRule.satisfyDistribution(FlinkConventions.STREAM_PHYSICAL(), localAgg, globalDistribution);
    RelTraitSet globalAggProvidedTraitSet = windowAgg.getTraitSet();
    // we put sliceEnd/windowEnd at the end of local output fields
    int endIndex = localAgg.getRowType().getFieldCount() - 1;
    final WindowingStrategy globalWindowing;
    if (windowing instanceof TimeAttributeWindowingStrategy) {
        globalWindowing = new SliceAttachedWindowingStrategy(windowing.getWindow(), windowing.getTimeAttributeType(), endIndex);
    } else {
        globalWindowing = new WindowAttachedWindowingStrategy(windowing.getWindow(), windowing.getTimeAttributeType(), endIndex);
    }
    StreamPhysicalGlobalWindowAggregate globalAgg = new StreamPhysicalGlobalWindowAggregate(windowAgg.getCluster(), globalAggProvidedTraitSet, newInput, realInput.getRowType(), globalGrouping, windowAgg.aggCalls(), globalWindowing, windowAgg.namedWindowProperties());
    call.transformTo(globalAgg);
}

Example 5 with FlinkRelDistribution

use of org.apache.flink.table.planner.plan.trait.FlinkRelDistribution in project flink by apache.

the class TwoStageOptimizedWindowAggregateRule method isInputSatisfyRequiredDistribution.

// ------------------------------------------------------------------------------------------
private boolean isInputSatisfyRequiredDistribution(RelNode input, int[] keys) {
    FlinkRelDistribution requiredDistribution = createDistribution(keys);
    FlinkRelDistribution inputDistribution = input.getTraitSet().getTrait(FlinkRelDistributionTraitDef.INSTANCE());
    return inputDistribution.satisfies(requiredDistribution);
}