Examples with Union org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union used on opensource projects

Example 6 with Union

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union in project hive by apache.

the class HiveAggregateIncrementalRewritingRuleBase method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Aggregate agg = call.rel(aggregateIndex);
    final Union union = call.rel(1);
    final RelBuilder relBuilder = call.builder();
    final RexBuilder rexBuilder = relBuilder.getRexBuilder();
    // 1) First branch is query, second branch is MV
    RelNode joinLeftInput = union.getInput(1);
    final T joinRightInput = createJoinRightInput(call);
    if (joinRightInput == null) {
        return;
    }
    // 2) Introduce a Project on top of MV scan having all columns from the view plus a boolean literal which indicates
    // whether the row with the key values coming from the joinRightInput exists in the view:
    // - true means exist
    // - null means not exists
    // Project also needed to encapsulate the view scan by a subquery -> this is required by
    // CalcitePlanner.fixUpASTAggregateInsertIncrementalRebuild
    // CalcitePlanner.fixUpASTAggregateInsertDeleteIncrementalRebuild
    List<RexNode> mvCols = new ArrayList<>(joinLeftInput.getRowType().getFieldCount());
    for (int i = 0; i < joinLeftInput.getRowType().getFieldCount(); ++i) {
        mvCols.add(rexBuilder.makeInputRef(joinLeftInput.getRowType().getFieldList().get(i).getType(), i));
    }
    mvCols.add(rexBuilder.makeLiteral(true));
    joinLeftInput = relBuilder.push(joinLeftInput).project(mvCols).build();
    // 3) Build conditions for join and start adding
    // expressions for project operator
    List<RexNode> projExprs = new ArrayList<>();
    List<RexNode> joinConjs = new ArrayList<>();
    int groupCount = agg.getGroupCount();
    int totalCount = agg.getGroupCount() + agg.getAggCallList().size();
    for (int leftPos = 0, rightPos = totalCount + 1; leftPos < groupCount; leftPos++, rightPos++) {
        RexNode leftRef = rexBuilder.makeInputRef(joinLeftInput.getRowType().getFieldList().get(leftPos).getType(), leftPos);
        RexNode rightRef = rexBuilder.makeInputRef(joinRightInput.rightInput.getRowType().getFieldList().get(leftPos).getType(), rightPos);
        projExprs.add(rightRef);
        RexNode nsEqExpr = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, ImmutableList.of(leftRef, rightRef));
        joinConjs.add(nsEqExpr);
    }
    // 4) Create join node
    RexNode joinCond = RexUtil.composeConjunction(rexBuilder, joinConjs);
    RelNode join = relBuilder.push(joinLeftInput).push(joinRightInput.rightInput).join(JoinRelType.RIGHT, joinCond).build();
    // functions
    for (int i = 0, leftPos = groupCount, rightPos = totalCount + 1 + groupCount; leftPos < totalCount; i++, leftPos++, rightPos++) {
        // case when source.s is null and mv2.s is null then null
        // case when source.s IS null then mv2.s
        // case when mv2.s IS null then source.s
        // else source.s + mv2.s end
        RexNode leftRef = rexBuilder.makeInputRef(joinLeftInput.getRowType().getFieldList().get(leftPos).getType(), leftPos);
        RexNode rightRef = rexBuilder.makeInputRef(joinRightInput.rightInput.getRowType().getFieldList().get(leftPos).getType(), rightPos);
        // Generate SQLOperator for merging the aggregations
        SqlAggFunction aggCall = agg.getAggCallList().get(i).getAggregation();
        RexNode elseReturn = createAggregateNode(aggCall, leftRef, rightRef, rexBuilder);
        // According to SQL standard (and Hive) Aggregate functions eliminates null values however operators used in
        // elseReturn expressions returns null if one of their operands is null
        // hence we need a null check of both operands.
        // Note: If both are null, we will fall into branch    WHEN leftNull THEN rightRef
        RexNode leftNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, leftRef);
        RexNode rightNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, rightRef);
        projExprs.add(rexBuilder.makeCall(SqlStdOperatorTable.CASE, leftNull, rightRef, rightNull, leftRef, elseReturn));
    }
    int flagIndex = joinLeftInput.getRowType().getFieldCount() - 1;
    RexNode flagNode = rexBuilder.makeInputRef(join.getRowType().getFieldList().get(flagIndex).getType(), flagIndex);
    // 6) Build plan
    RelNode newNode = relBuilder.push(join).filter(createFilterCondition(joinRightInput, flagNode, projExprs, relBuilder)).project(projExprs).build();
    call.transformTo(newNode);
}

Example 7 with Union

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union in project hive by apache.

the class HiveUnionPullUpConstantsRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Union union = call.rel(0);
    final int count = union.getRowType().getFieldCount();
    if (count == 1) {
        // Project operator.
        return;
    }
    final RexBuilder rexBuilder = union.getCluster().getRexBuilder();
    final RelMetadataQuery mq = call.getMetadataQuery();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(union);
    if (predicates == null) {
        return;
    }
    Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
    Map<RexNode, RexNode> constants = new HashMap<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        if (conditionsExtracted.containsKey(expr)) {
            constants.put(expr, conditionsExtracted.get(expr));
        }
    }
    // None of the expressions are constant. Nothing to do.
    if (constants.isEmpty()) {
        return;
    }
    // Create expressions for Project operators before and after the Union
    List<RelDataTypeField> fields = union.getRowType().getFieldList();
    List<RexNode> topChildExprs = new ArrayList<>();
    List<String> topChildExprsFields = new ArrayList<>();
    List<RexNode> refs = new ArrayList<>();
    ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        RelDataTypeField field = fields.get(i);
        if (constants.containsKey(expr)) {
            topChildExprs.add(constants.get(expr));
            topChildExprsFields.add(field.getName());
        } else {
            topChildExprs.add(expr);
            topChildExprsFields.add(field.getName());
            refs.add(expr);
            refsIndexBuilder.set(i);
        }
    }
    ImmutableBitSet refsIndex = refsIndexBuilder.build();
    // Update top Project positions
    final Mappings.TargetMapping mapping = RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse();
    topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
    // Create new Project-Union-Project sequences
    final RelBuilder relBuilder = call.builder();
    for (int i = 0; i < union.getInputs().size(); i++) {
        RelNode input = union.getInput(i);
        List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
        for (int j = 0; j < refsIndex.cardinality(); j++) {
            int pos = refsIndex.nth(j);
            newChildExprs.add(Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos), input.getRowType().getFieldList().get(pos).getName()));
        }
        if (newChildExprs.isEmpty()) {
            // At least a single item in project is required.
            newChildExprs.add(Pair.<RexNode, String>of(topChildExprs.get(0), topChildExprsFields.get(0)));
        }
        // Add the input with project on top
        relBuilder.push(input);
        relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
    }
    relBuilder.union(union.all, union.getInputs().size());
    // Create top Project fixing nullability of fields
    relBuilder.project(topChildExprs, topChildExprsFields);
    relBuilder.convert(union.getRowType(), false);
    call.transformTo(relBuilder.build());
}

Example 8 with Union

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union in project hive by apache.

the class HiveFilterSetOpTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
// implement RelOptRule
// We override the rule in order to do union all branch elimination
public void onMatch(RelOptRuleCall call) {
    Filter filterRel = call.rel(0);
    SetOp setOp = call.rel(1);
    RexNode condition = filterRel.getCondition();
    // create filters on top of each setop child, modifying the filter
    // condition to reference each setop child
    RexBuilder rexBuilder = filterRel.getCluster().getRexBuilder();
    final RelBuilder relBuilder = call.builder();
    List<RelDataTypeField> origFields = setOp.getRowType().getFieldList();
    int[] adjustments = new int[origFields.size()];
    final List<RelNode> newSetOpInputs = new ArrayList<>();
    RelNode lastInput = null;
    for (int index = 0; index < setOp.getInputs().size(); index++) {
        RelNode input = setOp.getInput(index);
        RexNode newCondition = condition.accept(new RelOptUtil.RexInputConverter(rexBuilder, origFields, input.getRowType().getFieldList(), adjustments));
        if (setOp instanceof Union && setOp.all) {
            final RelMetadataQuery mq = call.getMetadataQuery();
            final RelOptPredicateList predicates = mq.getPulledUpPredicates(input);
            if (predicates != null) {
                ImmutableList.Builder<RexNode> listBuilder = ImmutableList.builder();
                listBuilder.addAll(predicates.pulledUpPredicates);
                listBuilder.add(newCondition);
                RexExecutor executor = Util.first(filterRel.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR);
                final RexSimplify simplify = new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, executor);
                final RexNode cond = RexUtil.composeConjunction(rexBuilder, listBuilder.build());
                final RexNode x = simplify.simplifyUnknownAs(cond, RexUnknownAs.FALSE);
                if (x.isAlwaysFalse()) {
                    // branch so it won't read any data.
                    if (index == setOp.getInputs().size() - 1) {
                        lastInput = relBuilder.push(input).filter(newCondition).build();
                    }
                    // remove this branch
                    continue;
                }
            }
        }
        newSetOpInputs.add(relBuilder.push(input).filter(newCondition).build());
    }
    if (newSetOpInputs.size() > 1) {
        // create a new setop whose children are the filters created above
        SetOp newSetOp = setOp.copy(setOp.getTraitSet(), newSetOpInputs);
        call.transformTo(newSetOp);
    } else {
        // We have to keep at least a branch before we support empty values() in Hive
        RelNode result = newSetOpInputs.size() == 1 ? newSetOpInputs.get(0) : lastInput;
        call.transformTo(relBuilder.push(result).convert(filterRel.getRowType(), false).build());
    }
}

Example 9 with Union

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union in project calcite by apache.

the class EnumerableMergeUnionRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Sort sort = call.rel(0);
    final RelCollation collation = sort.getCollation();
    if (collation == null || collation.getFieldCollations().isEmpty()) {
        return false;
    }
    final Union union = call.rel(1);
    if (union.getInputs().size() < 2) {
        return false;
    }
    return true;
}

Example 10 with Union

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Union in project calcite by apache.

the class EnumerableMergeUnionRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Sort sort = call.rel(0);
    final RelCollation collation = sort.getCollation();
    final Union union = call.rel(1);
    final int unionInputsSize = union.getInputs().size();
    // Push down sort limit, if possible.
    RexNode inputFetch = null;
    if (sort.fetch != null) {
        if (sort.offset == null) {
            inputFetch = sort.fetch;
        } else if (sort.fetch instanceof RexLiteral && sort.offset instanceof RexLiteral) {
            inputFetch = call.builder().literal(RexLiteral.intValue(sort.fetch) + RexLiteral.intValue(sort.offset));
        }
    }
    final List<RelNode> inputs = new ArrayList<>(unionInputsSize);
    for (RelNode input : union.getInputs()) {
        final RelNode newInput = sort.copy(sort.getTraitSet(), input, collation, null, inputFetch);
        inputs.add(convert(newInput, newInput.getTraitSet().replace(EnumerableConvention.INSTANCE)));
    }
    RelNode result = EnumerableMergeUnion.create(sort.getCollation(), inputs, union.all);
    // The output of the MergeUnion is already sorted, so we do not need a sort anymore.
    if (sort.offset != null || sort.fetch != null) {
        result = EnumerableLimit.create(result, sort.offset, sort.fetch);
    }
    call.transformTo(result);
}