Examples with RelCollation org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation used on opensource projects

Example 31 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project hive by apache.

the class HiveRelDecorrelator method decorrelateRel.

/**
 * Rewrite Sort.
 *
 * @param rel Sort to be rewritten
 */
public Frame decorrelateRel(HiveSortLimit rel) {
    // Sort itself should not reference cor vars.
    assert !cm.mapRefRelToCorRef.containsKey(rel);
    // Sort only references field positions in collations field.
    // The collations field in the newRel now need to refer to the
    // new output positions in its input.
    // Its output does not change the input ordering, so there's no
    // need to call propagateExpr.
    final RelNode oldInput = rel.getInput();
    final Frame frame = getInvoke(oldInput, rel);
    if (frame == null) {
        // If input has not been rewritten, do not rewrite this rel.
        return null;
    }
    final RelNode newInput = frame.r;
    Mappings.TargetMapping mapping = Mappings.target(frame.oldToNewOutputs, oldInput.getRowType().getFieldCount(), newInput.getRowType().getFieldCount());
    RelCollation oldCollation = rel.getCollation();
    RelCollation newCollation = RexUtil.apply(mapping, oldCollation);
    final RelNode newSort = HiveSortLimit.create(newInput, newCollation, rel.offset, rel.fetch);
    // Sort does not change input ordering
    return register(rel, newSort, frame.oldToNewOutputs, frame.corDefOutputs);
}

Example 32 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project drill by axbaretto.

the class StreamAggPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillAggregateRel aggregate = call.rel(0);
    RelNode input = aggregate.getInput();
    final RelCollation collation = getCollation(aggregate);
    RelTraitSet traits;
    if (aggregate.containsDistinctCall()) {
        // currently, don't use StreamingAggregate if any of the logical aggrs contains DISTINCT
        return;
    }
    try {
        if (aggregate.getGroupSet().isEmpty()) {
            DrillDistributionTrait singleDist = DrillDistributionTrait.SINGLETON;
            final RelTraitSet singleDistTrait = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(singleDist);
            if (create2PhasePlan(call, aggregate)) {
                traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
                RelNode convertedInput = convert(input, traits);
                new SubsetTransformer<DrillAggregateRel, InvalidRelException>(call) {

                    @Override
                    public RelNode convertChild(final DrillAggregateRel join, final RelNode rel) throws InvalidRelException {
                        DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
                        RelTraitSet traits = newTraitSet(Prel.DRILL_PHYSICAL, toDist);
                        RelNode newInput = convert(rel, traits);
                        StreamAggPrel phase1Agg = new StreamAggPrel(aggregate.getCluster(), traits, newInput, aggregate.indicator, aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList(), OperatorPhase.PHASE_1of2);
                        UnionExchangePrel exch = new UnionExchangePrel(phase1Agg.getCluster(), singleDistTrait, phase1Agg);
                        ImmutableBitSet newGroupSet = remapGroupSet(aggregate.getGroupSet());
                        List<ImmutableBitSet> newGroupSets = Lists.newArrayList();
                        for (ImmutableBitSet groupSet : aggregate.getGroupSets()) {
                            newGroupSets.add(remapGroupSet(groupSet));
                        }
                        return new StreamAggPrel(aggregate.getCluster(), singleDistTrait, exch, aggregate.indicator, newGroupSet, newGroupSets, phase1Agg.getPhase2AggCalls(), OperatorPhase.PHASE_2of2);
                    }
                }.go(aggregate, convertedInput);
            } else {
                createTransformRequest(call, aggregate, input, singleDistTrait);
            }
        } else {
            // hash distribute on all grouping keys
            final DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true)));
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(distOnAllKeys);
            createTransformRequest(call, aggregate, input, traits);
            // hash distribute on one grouping key
            DrillDistributionTrait distOnOneKey = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, false)));
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(distOnOneKey);
            if (create2PhasePlan(call, aggregate)) {
                traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
                RelNode convertedInput = convert(input, traits);
                new SubsetTransformer<DrillAggregateRel, InvalidRelException>(call) {

                    @Override
                    public RelNode convertChild(final DrillAggregateRel aggregate, final RelNode rel) throws InvalidRelException {
                        DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
                        RelTraitSet traits = newTraitSet(Prel.DRILL_PHYSICAL, collation, toDist);
                        RelNode newInput = convert(rel, traits);
                        StreamAggPrel phase1Agg = new StreamAggPrel(aggregate.getCluster(), traits, newInput, aggregate.indicator, aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList(), OperatorPhase.PHASE_1of2);
                        int numEndPoints = PrelUtil.getSettings(phase1Agg.getCluster()).numEndPoints();
                        HashToMergeExchangePrel exch = new HashToMergeExchangePrel(phase1Agg.getCluster(), phase1Agg.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distOnAllKeys), phase1Agg, ImmutableList.copyOf(getDistributionField(aggregate, true)), collation, numEndPoints);
                        ImmutableBitSet newGroupSet = remapGroupSet(aggregate.getGroupSet());
                        List<ImmutableBitSet> newGroupSets = Lists.newArrayList();
                        for (ImmutableBitSet groupSet : aggregate.getGroupSets()) {
                            newGroupSets.add(remapGroupSet(groupSet));
                        }
                        return new StreamAggPrel(aggregate.getCluster(), exch.getTraitSet(), exch, aggregate.indicator, newGroupSet, newGroupSets, phase1Agg.getPhase2AggCalls(), OperatorPhase.PHASE_2of2);
                    }
                }.go(aggregate, convertedInput);
            }
        }
    } catch (InvalidRelException e) {
        tracer.warn(e.toString());
    }
}

Example 33 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project drill by axbaretto.

the class WindowPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillWindowRel window = call.rel(0);
    RelNode input = call.rel(1);
    // TODO: Order window based on existing partition by
    // input.getTraitSet().subsumes()
    boolean partitionby = false;
    boolean addMerge = false;
    // The start index of the constant fields of DrillWindowRel
    final int startConstantsIndex = window.getInput().getRowType().getFieldCount();
    int constantShiftIndex = 0;
    for (final Ord<Window.Group> w : Ord.zip(window.groups)) {
        Window.Group windowBase = w.getValue();
        RelTraitSet traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
        // For empty Over-Clause
        if (windowBase.keys.isEmpty() && windowBase.orderKeys.getFieldCollations().isEmpty()) {
            DrillDistributionTrait distEmptyKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.SINGLETON);
            traits = traits.plus(distEmptyKeys);
        } else if (windowBase.keys.size() > 0) {
            DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(windowBase)));
            partitionby = true;
            traits = traits.plus(distOnAllKeys);
        } else if (windowBase.orderKeys.getFieldCollations().size() > 0) {
            // if only the order-by clause is specified, there is a single partition
            // consisting of all the rows, so we do a distributed sort followed by a
            // single merge as the input of the window operator
            DrillDistributionTrait distKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFieldsFromCollation(windowBase)));
            traits = traits.plus(distKeys);
            if (!isSingleMode(call)) {
                addMerge = true;
            }
        }
        // Add collation trait if either partition-by or order-by is specified.
        if (partitionby || windowBase.orderKeys.getFieldCollations().size() > 0) {
            RelCollation collation = getCollation(windowBase);
            traits = traits.plus(collation);
        }
        RelNode convertedInput = convert(input, traits);
        if (addMerge) {
            traits = traits.plus(DrillDistributionTrait.SINGLETON);
            convertedInput = new SingleMergeExchangePrel(window.getCluster(), traits, convertedInput, windowBase.collation());
        }
        List<RelDataTypeField> newRowFields = Lists.newArrayList();
        for (RelDataTypeField field : convertedInput.getRowType().getFieldList()) {
            newRowFields.add(field);
        }
        Iterable<RelDataTypeField> newWindowFields = Iterables.filter(window.getRowType().getFieldList(), new Predicate<RelDataTypeField>() {

            @Override
            public boolean apply(RelDataTypeField relDataTypeField) {
                return relDataTypeField.getName().startsWith("w" + w.i + "$");
            }
        });
        for (RelDataTypeField newField : newWindowFields) {
            newRowFields.add(newField);
        }
        RelDataType rowType = new RelRecordType(newRowFields);
        List<Window.RexWinAggCall> newWinAggCalls = Lists.newArrayList();
        for (Ord<Window.RexWinAggCall> aggOrd : Ord.zip(windowBase.aggCalls)) {
            Window.RexWinAggCall aggCall = aggOrd.getValue();
            // If the argument points at the constant and
            // additional fields have been generated by the Window below,
            // the index of constants will be shifted
            final List<RexNode> newOperandsOfWindowFunction = Lists.newArrayList();
            for (RexNode operand : aggCall.getOperands()) {
                if (operand instanceof RexInputRef) {
                    final RexInputRef rexInputRef = (RexInputRef) operand;
                    final int refIndex = rexInputRef.getIndex();
                    // Check if this RexInputRef points at the constants
                    if (rexInputRef.getIndex() >= startConstantsIndex) {
                        operand = new RexInputRef(refIndex + constantShiftIndex, window.constants.get(refIndex - startConstantsIndex).getType());
                    }
                }
                newOperandsOfWindowFunction.add(operand);
            }
            aggCall = new Window.RexWinAggCall((SqlAggFunction) aggCall.getOperator(), aggCall.getType(), newOperandsOfWindowFunction, aggCall.ordinal, aggCall.distinct);
            newWinAggCalls.add(new Window.RexWinAggCall((SqlAggFunction) aggCall.getOperator(), aggCall.getType(), aggCall.getOperands(), aggOrd.i, aggCall.distinct));
        }
        windowBase = new Window.Group(windowBase.keys, windowBase.isRows, windowBase.lowerBound, windowBase.upperBound, windowBase.orderKeys, newWinAggCalls);
        input = new WindowPrel(window.getCluster(), window.getTraitSet().merge(traits), convertedInput, window.getConstants(), rowType, windowBase);
        constantShiftIndex += windowBase.aggCalls.size();
    }
    call.transformTo(input);
}

Example 34 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project drill by axbaretto.

the class WriterPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillWriterRel writer = call.rel(0);
    final RelNode input = call.rel(1);
    final List<Integer> keys = writer.getPartitionKeys();
    final RelCollation collation = getCollation(keys);
    final boolean hashDistribute = PrelUtil.getPlannerSettings(call.getPlanner()).getOptions().getOption(ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR);
    final RelTraitSet traits = hashDistribute ? input.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(getDistribution(keys)) : input.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation);
    final RelNode convertedInput = convert(input, traits);
    if (!new WriteTraitPull(call).go(writer, convertedInput)) {
        DrillWriterRelBase newWriter = new WriterPrel(writer.getCluster(), convertedInput.getTraitSet(), convertedInput, writer.getCreateTableEntry());
        call.transformTo(newWriter);
    }
}

Example 35 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project hive by apache.

the class RelFieldTrimmer method trimFields.

/**
 * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
 * {@link org.apache.calcite.rel.core.Sort}.
 */
public TrimResult trimFields(Sort sort, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
    final RelDataType rowType = sort.getRowType();
    final int fieldCount = rowType.getFieldCount();
    final RelCollation collation = sort.getCollation();
    final RelNode input = sort.getInput();
    // We use the fields used by the consumer, plus any fields used as sort
    // keys.
    final ImmutableBitSet.Builder inputFieldsUsed = fieldsUsed.rebuild();
    for (RelFieldCollation field : collation.getFieldCollations()) {
        inputFieldsUsed.set(field.getFieldIndex());
    }
    // Create input with trimmed columns.
    final Set<RelDataTypeField> inputExtraFields = Collections.emptySet();
    TrimResult trimResult = trimChild(sort, input, inputFieldsUsed.build(), inputExtraFields);
    RelNode newInput = trimResult.left;
    final Mapping inputMapping = trimResult.right;
    // there's nothing we can do.
    if (newInput == input && inputMapping.isIdentity() && fieldsUsed.cardinality() == fieldCount) {
        return result(sort, Mappings.createIdentity(fieldCount));
    }
    // leave the Sort unchanged in case we have dynamic limits
    if (sort.offset instanceof RexDynamicParam || sort.fetch instanceof RexDynamicParam) {
        return result(sort, inputMapping);
    }
    final RelBuilder relBuilder = REL_BUILDER.get();
    relBuilder.push(newInput);
    final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
    final int fetch = sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch);
    final ImmutableList<RexNode> fields = relBuilder.fields(RexUtil.apply(inputMapping, collation));
    relBuilder.sortLimit(offset, fetch, fields);
    // needs them for its condition.
    return result(relBuilder.build(), inputMapping);
}