Examples with RelFieldCollation org.apache.calcite.rel.RelFieldCollation used on opensource projects

Example 11 with RelFieldCollation

use of org.apache.calcite.rel.RelFieldCollation in project hive by apache.

the class HiveRelColumnsAlignment method align.

public RelNode align(Join rel, List<RelFieldCollation> collations) {
    ImmutableList.Builder<RelFieldCollation> propagateCollationsLeft = ImmutableList.builder();
    ImmutableList.Builder<RelFieldCollation> propagateCollationsRight = ImmutableList.builder();
    final int nLeftColumns = rel.getLeft().getRowType().getFieldList().size();
    Map<Integer, RexNode> idxToConjuncts = new HashMap<>();
    Map<Integer, Integer> refToRef = new HashMap<>();
    // 1) We extract the conditions that can be useful
    List<RexNode> conjuncts = new ArrayList<>();
    List<RexNode> otherConjuncts = new ArrayList<>();
    for (RexNode conj : RelOptUtil.conjunctions(rel.getCondition())) {
        if (conj.getKind() != SqlKind.EQUALS) {
            otherConjuncts.add(conj);
            continue;
        }
        // TODO: Currently we only support EQUAL operator on two references.
        // We might extend the logic to support other (order-preserving)
        // UDFs here.
        RexCall equals = (RexCall) conj;
        if (!(equals.getOperands().get(0) instanceof RexInputRef) || !(equals.getOperands().get(1) instanceof RexInputRef)) {
            otherConjuncts.add(conj);
            continue;
        }
        RexInputRef ref0 = (RexInputRef) equals.getOperands().get(0);
        RexInputRef ref1 = (RexInputRef) equals.getOperands().get(1);
        if ((ref0.getIndex() < nLeftColumns && ref1.getIndex() >= nLeftColumns) || (ref1.getIndex() < nLeftColumns && ref0.getIndex() >= nLeftColumns)) {
            // We made sure the references are for different join inputs
            idxToConjuncts.put(ref0.getIndex(), equals);
            idxToConjuncts.put(ref1.getIndex(), equals);
            refToRef.put(ref0.getIndex(), ref1.getIndex());
            refToRef.put(ref1.getIndex(), ref0.getIndex());
        } else {
            otherConjuncts.add(conj);
        }
    }
    // that we will propagate to the inputs of the join
    for (RelFieldCollation c : collations) {
        RexNode equals = idxToConjuncts.get(c.getFieldIndex());
        if (equals != null) {
            conjuncts.add(equals);
            idxToConjuncts.remove(c.getFieldIndex());
            idxToConjuncts.remove(refToRef.get(c.getFieldIndex()));
            if (c.getFieldIndex() < nLeftColumns) {
                propagateCollationsLeft.add(c.copy(c.getFieldIndex()));
                propagateCollationsRight.add(c.copy(refToRef.get(c.getFieldIndex()) - nLeftColumns));
            } else {
                propagateCollationsLeft.add(c.copy(refToRef.get(c.getFieldIndex())));
                propagateCollationsRight.add(c.copy(c.getFieldIndex() - nLeftColumns));
            }
        }
    }
    final Set<RexNode> visited = new HashSet<>();
    for (Entry<Integer, RexNode> e : idxToConjuncts.entrySet()) {
        if (visited.add(e.getValue())) {
            // Not included in the input collations, but can be propagated as this Join
            // might enforce it
            conjuncts.add(e.getValue());
            if (e.getKey() < nLeftColumns) {
                propagateCollationsLeft.add(new RelFieldCollation(e.getKey()));
                propagateCollationsRight.add(new RelFieldCollation(refToRef.get(e.getKey()) - nLeftColumns));
            } else {
                propagateCollationsLeft.add(new RelFieldCollation(refToRef.get(e.getKey())));
                propagateCollationsRight.add(new RelFieldCollation(e.getKey() - nLeftColumns));
            }
        }
    }
    conjuncts.addAll(otherConjuncts);
    // 3) We propagate
    final RelNode newLeftInput = dispatchAlign(rel.getLeft(), propagateCollationsLeft.build());
    final RelNode newRightInput = dispatchAlign(rel.getRight(), propagateCollationsRight.build());
    // 4) We change the Join operator to reflect this info
    final RelNode newJoin = rel.copy(rel.getTraitSet(), RexUtil.composeConjunction(relBuilder.getRexBuilder(), conjuncts, false), newLeftInput, newRightInput, rel.getJoinType(), rel.isSemiJoinDone());
    return newJoin;
}

Example 12 with RelFieldCollation

use of org.apache.calcite.rel.RelFieldCollation in project hive by apache.

the class HiveSortProjectTransposeRule method onMatch.

// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit sort = call.rel(0);
    final HiveProject project = call.rel(1);
    // Determine mapping between project input and output fields. If sort
    // relies on non-trivial expressions, we can't push.
    final Mappings.TargetMapping map = RelOptUtil.permutation(project.getProjects(), project.getInput().getRowType());
    for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
        if (map.getTargetOpt(fc.getFieldIndex()) < 0) {
            return;
        }
    }
    // Create new collation
    final RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RexUtil.apply(map, sort.getCollation()));
    // New operators
    final HiveSortLimit newSort = sort.copy(sort.getTraitSet().replace(newCollation), project.getInput(), newCollation, sort.offset, sort.fetch);
    final RelNode newProject = project.copy(sort.getTraitSet(), ImmutableList.<RelNode>of(newSort));
    call.transformTo(newProject);
}

Example 13 with RelFieldCollation

use of org.apache.calcite.rel.RelFieldCollation in project hive by apache.

the class HiveInsertExchange4JoinRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    JoinPredicateInfo joinPredInfo;
    if (call.rel(0) instanceof HiveMultiJoin) {
        HiveMultiJoin multiJoin = call.rel(0);
        joinPredInfo = multiJoin.getJoinPredicateInfo();
    } else if (call.rel(0) instanceof HiveJoin) {
        HiveJoin hiveJoin = call.rel(0);
        joinPredInfo = hiveJoin.getJoinPredicateInfo();
    } else if (call.rel(0) instanceof Join) {
        Join join = call.rel(0);
        try {
            joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(join);
        } catch (CalciteSemanticException e) {
            throw new RuntimeException(e);
        }
    } else {
        return;
    }
    for (RelNode child : call.rel(0).getInputs()) {
        if (((HepRelVertex) child).getCurrentRel() instanceof Exchange) {
            return;
        }
    }
    // Get key columns from inputs. Those are the columns on which we will distribute on.
    // It is also the columns we will sort on.
    List<RelNode> newInputs = new ArrayList<RelNode>();
    for (int i = 0; i < call.rel(0).getInputs().size(); i++) {
        List<Integer> joinKeyPositions = new ArrayList<Integer>();
        ImmutableList.Builder<RexNode> joinExprsBuilder = new ImmutableList.Builder<RexNode>();
        Set<String> keySet = Sets.newHashSet();
        ImmutableList.Builder<RelFieldCollation> collationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
        for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
            JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo.getEquiJoinPredicateElements().get(j);
            for (RexNode joinExprNode : joinLeafPredInfo.getJoinExprs(i)) {
                if (keySet.add(joinExprNode.toString())) {
                    joinExprsBuilder.add(joinExprNode);
                }
            }
            for (int pos : joinLeafPredInfo.getProjsJoinKeysInChildSchema(i)) {
                if (!joinKeyPositions.contains(pos)) {
                    joinKeyPositions.add(pos);
                    collationListBuilder.add(new RelFieldCollation(pos));
                }
            }
        }
        HiveSortExchange exchange = HiveSortExchange.create(call.rel(0).getInput(i), new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, joinKeyPositions), new HiveRelCollation(collationListBuilder.build()), joinExprsBuilder.build());
        newInputs.add(exchange);
    }
    RelNode newOp;
    if (call.rel(0) instanceof HiveMultiJoin) {
        HiveMultiJoin multiJoin = call.rel(0);
        newOp = multiJoin.copy(multiJoin.getTraitSet(), newInputs);
    } else if (call.rel(0) instanceof Join) {
        Join join = call.rel(0);
        newOp = join.copy(join.getTraitSet(), join.getCondition(), newInputs.get(0), newInputs.get(1), join.getJoinType(), join.isSemiJoinDone());
    } else {
        return;
    }
    call.getPlanner().onCopy(call.rel(0), newOp);
    call.transformTo(newOp);
}

Example 14 with RelFieldCollation

use of org.apache.calcite.rel.RelFieldCollation in project hive by apache.

the class HiveAlgorithmsUtil method getJoinCollation.

public static ImmutableList<RelCollation> getJoinCollation(JoinPredicateInfo joinPredInfo, MapJoinStreamingRelation streamingRelation) {
    // Compute collations
    ImmutableList.Builder<RelFieldCollation> collationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
    ImmutableList.Builder<RelFieldCollation> leftCollationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
    ImmutableList.Builder<RelFieldCollation> rightCollationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
    for (int i = 0; i < joinPredInfo.getEquiJoinPredicateElements().size(); i++) {
        JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo.getEquiJoinPredicateElements().get(i);
        for (int leftPos : joinLeafPredInfo.getProjsFromLeftPartOfJoinKeysInJoinSchema()) {
            final RelFieldCollation leftFieldCollation = new RelFieldCollation(leftPos);
            collationListBuilder.add(leftFieldCollation);
            leftCollationListBuilder.add(leftFieldCollation);
        }
        for (int rightPos : joinLeafPredInfo.getProjsFromRightPartOfJoinKeysInJoinSchema()) {
            final RelFieldCollation rightFieldCollation = new RelFieldCollation(rightPos);
            collationListBuilder.add(rightFieldCollation);
            rightCollationListBuilder.add(rightFieldCollation);
        }
    }
    // Return join collations
    final ImmutableList<RelCollation> collation;
    switch(streamingRelation) {
        case LEFT_RELATION:
            collation = ImmutableList.of(RelCollationTraitDef.INSTANCE.canonize(new HiveRelCollation(leftCollationListBuilder.build())));
            break;
        case RIGHT_RELATION:
            collation = ImmutableList.of(RelCollationTraitDef.INSTANCE.canonize(new HiveRelCollation(rightCollationListBuilder.build())));
            break;
        default:
            collation = ImmutableList.of(RelCollationTraitDef.INSTANCE.canonize(new HiveRelCollation(collationListBuilder.build())));
            break;
    }
    return collation;
}

Example 15 with RelFieldCollation

use of org.apache.calcite.rel.RelFieldCollation in project hive by apache.

the class HiveRelColumnsAlignment method align.

public RelNode align(Aggregate rel, List<RelFieldCollation> collations) {
    // 1) We extract the group by positions that are part of the collations and
    // sort them so they respect it
    LinkedHashSet<Integer> aggregateColumnsOrder = new LinkedHashSet<>();
    ImmutableList.Builder<RelFieldCollation> propagateCollations = ImmutableList.builder();
    if (!rel.indicator && !collations.isEmpty()) {
        for (RelFieldCollation c : collations) {
            if (c.getFieldIndex() < rel.getGroupCount()) {
                // Group column found
                if (aggregateColumnsOrder.add(c.getFieldIndex())) {
                    propagateCollations.add(c.copy(rel.getGroupSet().nth(c.getFieldIndex())));
                }
            }
        }
    }
    for (int i = 0; i < rel.getGroupCount(); i++) {
        if (!aggregateColumnsOrder.contains(i)) {
            // Not included in the input collations, but can be propagated as this Aggregate
            // will enforce it
            propagateCollations.add(new RelFieldCollation(rel.getGroupSet().nth(i)));
        }
    }
    // 2) We propagate
    final RelNode child = dispatchAlign(rel.getInput(), propagateCollations.build());
    // 3) We annotate the Aggregate operator with this info
    final HiveAggregate newAggregate = (HiveAggregate) rel.copy(rel.getTraitSet(), ImmutableList.of(child));
    newAggregate.setAggregateColumnsOrder(aggregateColumnsOrder);
    return newAggregate;
}