Search in sources :

Example 81 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project flink by apache.

the class StreamPhysicalPythonOverAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalOverAggregate logicWindow = (FlinkLogicalOverAggregate) rel;
    if (logicWindow.groups.size() > 1) {
        throw new TableException("Over Agg: Unsupported use of OVER windows. " + "All aggregates must be computed on the same window. " + "please re-check the over window statement.");
    }
    ImmutableBitSet keys = logicWindow.groups.get(0).keys;
    FlinkRelDistribution requiredDistribution;
    if (!keys.isEmpty()) {
        requiredDistribution = FlinkRelDistribution.hash(keys.asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelNode input = logicWindow.getInput();
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    return new StreamPhysicalPythonOverAggregate(rel.getCluster(), providedTraitSet, newInput, rel.getRowType(), logicWindow);
}
Also used : TableException(org.apache.flink.table.api.TableException) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) ImmutableBitSet(org.apache.calcite.util.ImmutableBitSet) RelNode(org.apache.calcite.rel.RelNode) FlinkLogicalOverAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalOverAggregate) StreamPhysicalPythonOverAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalPythonOverAggregate) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Example 82 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project flink by apache.

the class StreamPhysicalPythonGroupAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalAggregate agg = (FlinkLogicalAggregate) rel;
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = rel.getCluster().getPlanner().emptyTraitSet().replace(requiredDistribution).replace(FlinkConventions.STREAM_PHYSICAL());
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(agg.getInput(), requiredTraitSet);
    return new StreamPhysicalPythonGroupAggregate(rel.getCluster(), providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(agg.getAggCallList()));
}
Also used : FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) RelNode(org.apache.calcite.rel.RelNode) FlinkLogicalAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalAggregate) RelTraitSet(org.apache.calcite.plan.RelTraitSet) StreamPhysicalPythonGroupAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalPythonGroupAggregate)

Example 83 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalWindowAggregate agg = (FlinkLogicalWindowAggregate) rel;
    LogicalWindow window = agg.getWindow();
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean isPandasPythonUDAF = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    if (isPandasPythonUDAF && window instanceof SessionGroupWindow) {
        throw new TableException("Session Group Window is currently not supported for Pandas UDAF.");
    }
    RelNode input = agg.getInput();
    RelOptCluster cluster = rel.getCluster();
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    ReadableConfig config = ShortcutUtils.unwrapTableConfig(rel);
    WindowEmitStrategy emitStrategy = WindowEmitStrategy.apply(config, agg.getWindow());
    if (emitStrategy.produceUpdates()) {
        throw new TableException("Python Group Window Aggregate Function is currently not supported for early fired or lately fired.");
    }
    return new StreamPhysicalPythonGroupWindowAggregate(cluster, providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(aggCalls), agg.getWindow(), agg.getNamedProperties(), emitStrategy);
}
Also used : RelOptCluster(org.apache.calcite.plan.RelOptCluster) StreamPhysicalPythonGroupWindowAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalPythonGroupWindowAggregate) TableException(org.apache.flink.table.api.TableException) RelTraitSet(org.apache.calcite.plan.RelTraitSet) AggregateCall(org.apache.calcite.rel.core.AggregateCall) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) ReadableConfig(org.apache.flink.configuration.ReadableConfig) LogicalWindow(org.apache.flink.table.planner.plan.logical.LogicalWindow) RelNode(org.apache.calcite.rel.RelNode) WindowEmitStrategy(org.apache.flink.table.planner.plan.utils.WindowEmitStrategy) FlinkLogicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalWindowAggregate) SessionGroupWindow(org.apache.flink.table.planner.plan.logical.SessionGroupWindow)

Example 84 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project flink by apache.

the class TwoStageOptimizedWindowAggregateRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final StreamPhysicalWindowAggregate windowAgg = call.rel(0);
    final RelNode realInput = call.rel(2);
    final WindowingStrategy windowing = windowAgg.windowing();
    RelTraitSet localTraitSet = realInput.getTraitSet().plus(ModifyKindSetTrait.INSERT_ONLY()).plus(UpdateKindTrait.NONE());
    StreamPhysicalLocalWindowAggregate localAgg = new StreamPhysicalLocalWindowAggregate(windowAgg.getCluster(), localTraitSet, realInput, windowAgg.grouping(), windowAgg.aggCalls(), windowing);
    // grouping keys is forwarded by local agg, use indices instead of groupings
    int[] globalGrouping = IntStream.range(0, windowAgg.grouping().length).toArray();
    FlinkRelDistribution globalDistribution = createDistribution(globalGrouping);
    // create exchange if needed
    RelNode newInput = FlinkExpandConversionRule.satisfyDistribution(FlinkConventions.STREAM_PHYSICAL(), localAgg, globalDistribution);
    RelTraitSet globalAggProvidedTraitSet = windowAgg.getTraitSet();
    // we put sliceEnd/windowEnd at the end of local output fields
    int endIndex = localAgg.getRowType().getFieldCount() - 1;
    final WindowingStrategy globalWindowing;
    if (windowing instanceof TimeAttributeWindowingStrategy) {
        globalWindowing = new SliceAttachedWindowingStrategy(windowing.getWindow(), windowing.getTimeAttributeType(), endIndex);
    } else {
        globalWindowing = new WindowAttachedWindowingStrategy(windowing.getWindow(), windowing.getTimeAttributeType(), endIndex);
    }
    StreamPhysicalGlobalWindowAggregate globalAgg = new StreamPhysicalGlobalWindowAggregate(windowAgg.getCluster(), globalAggProvidedTraitSet, newInput, realInput.getRowType(), globalGrouping, windowAgg.aggCalls(), globalWindowing, windowAgg.namedWindowProperties());
    call.transformTo(globalAgg);
}
Also used : StreamPhysicalLocalWindowAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLocalWindowAggregate) FlinkRelDistribution(org.apache.flink.table.planner.plan.trait.FlinkRelDistribution) RelNode(org.apache.calcite.rel.RelNode) StreamPhysicalWindowAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowAggregate) TimeAttributeWindowingStrategy(org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy) SliceAttachedWindowingStrategy(org.apache.flink.table.planner.plan.logical.SliceAttachedWindowingStrategy) WindowAttachedWindowingStrategy(org.apache.flink.table.planner.plan.logical.WindowAttachedWindowingStrategy) RelTraitSet(org.apache.calcite.plan.RelTraitSet) StreamPhysicalGlobalWindowAggregate(org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalGlobalWindowAggregate) WindowAttachedWindowingStrategy(org.apache.flink.table.planner.plan.logical.WindowAttachedWindowingStrategy) TimeAttributeWindowingStrategy(org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy) WindowingStrategy(org.apache.flink.table.planner.plan.logical.WindowingStrategy) SliceAttachedWindowingStrategy(org.apache.flink.table.planner.plan.logical.SliceAttachedWindowingStrategy)

Example 85 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project beam by apache.

the class ZetaSQLQueryPlanner method convertToBeamRelInternal.

private BeamRelNode convertToBeamRelInternal(String sql, QueryParameters queryParams) {
    RelRoot root = plannerImpl.rel(sql, queryParams);
    RelTraitSet desiredTraits = root.rel.getTraitSet().replace(BeamLogicalConvention.INSTANCE).replace(root.collation).simplify();
    // beam physical plan
    root.rel.getCluster().setMetadataProvider(ChainedRelMetadataProvider.of(org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableList.of(NonCumulativeCostImpl.SOURCE, RelMdNodeStats.SOURCE, root.rel.getCluster().getMetadataProvider())));
    root.rel.getCluster().setMetadataQuerySupplier(BeamRelMetadataQuery::instance);
    RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(root.rel.getCluster().getMetadataProvider()));
    root.rel.getCluster().invalidateMetadataQuery();
    try {
        BeamRelNode beamRelNode = (BeamRelNode) plannerImpl.transform(0, desiredTraits, root.rel);
        LOG.info("BEAMPlan>\n" + RelOptUtil.toString(beamRelNode));
        return beamRelNode;
    } catch (RelOptPlanner.CannotPlanException e) {
        throw new SqlConversionException("Failed to produce plan for query " + sql, e);
    }
}
Also used : BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamRelMetadataQuery(org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelMetadataQuery) RelRoot(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelRoot) RelTraitSet(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet) SqlConversionException(org.apache.beam.sdk.extensions.sql.impl.SqlConversionException) RelOptPlanner(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptPlanner)

Aggregations

RelTraitSet (org.apache.calcite.plan.RelTraitSet)190 RelNode (org.apache.calcite.rel.RelNode)111 RelOptCluster (org.apache.calcite.plan.RelOptCluster)38 RelCollation (org.apache.calcite.rel.RelCollation)36 RelOptPlanner (org.apache.calcite.plan.RelOptPlanner)26 RexNode (org.apache.calcite.rex.RexNode)24 ArrayList (java.util.ArrayList)20 InvalidRelException (org.apache.calcite.rel.InvalidRelException)19 RelDataType (org.apache.calcite.rel.type.RelDataType)14 SqlNode (org.apache.calcite.sql.SqlNode)14 List (java.util.List)13 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)13 RelOptTable (org.apache.calcite.plan.RelOptTable)11 Sort (org.apache.calcite.rel.core.Sort)11 RelMetadataQuery (org.apache.calcite.rel.metadata.RelMetadataQuery)11 ImmutableList (com.google.common.collect.ImmutableList)10 RelFieldCollation (org.apache.calcite.rel.RelFieldCollation)10 Test (org.junit.Test)9 Table (org.apache.calcite.schema.Table)8 RexInputRef (org.apache.calcite.rex.RexInputRef)7