Examples with RelNode org.apache.calcite.rel.RelNode used on opensource projects

Example 36 with RelNode

use of org.apache.calcite.rel.RelNode in project hive by apache.

the class HiveRelDecorrelator method decorrelateRel.

public Frame decorrelateRel(HiveJoin rel) throws SemanticException {
    //
    // Rewrite logic:
    //
    // 1. rewrite join condition.
    // 2. map output positions and produce cor vars if any.
    //
    final RelNode oldLeft = rel.getInput(0);
    final RelNode oldRight = rel.getInput(1);
    final Frame leftFrame = getInvoke(oldLeft, rel);
    final Frame rightFrame = getInvoke(oldRight, rel);
    if (leftFrame == null || rightFrame == null) {
        // If any input has not been rewritten, do not rewrite this rel.
        return null;
    }
    final RelNode newJoin = HiveJoin.getJoin(rel.getCluster(), leftFrame.r, rightFrame.r, decorrelateExpr(rel.getCondition()), rel.getJoinType());
    // Create the mapping between the output of the old correlation rel
    // and the new join rel
    Map<Integer, Integer> mapOldToNewOutputs = Maps.newHashMap();
    int oldLeftFieldCount = oldLeft.getRowType().getFieldCount();
    int newLeftFieldCount = leftFrame.r.getRowType().getFieldCount();
    int oldRightFieldCount = oldRight.getRowType().getFieldCount();
    assert rel.getRowType().getFieldCount() == oldLeftFieldCount + oldRightFieldCount;
    // Left input positions are not changed.
    mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs);
    // Right input positions are shifted by newLeftFieldCount.
    for (int i = 0; i < oldRightFieldCount; i++) {
        mapOldToNewOutputs.put(i + oldLeftFieldCount, rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount);
    }
    final SortedMap<CorDef, Integer> corDefOutputs = new TreeMap<>(leftFrame.corDefOutputs);
    // Right input positions are shifted by newLeftFieldCount.
    for (Map.Entry<CorDef, Integer> entry : rightFrame.corDefOutputs.entrySet()) {
        corDefOutputs.put(entry.getKey(), entry.getValue() + newLeftFieldCount);
    }
    return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs);
}

Example 37 with RelNode

use of org.apache.calcite.rel.RelNode in project hive by apache.

the class HiveRelDecorrelator method createValueGenerator.

/**
   * Create RelNode tree that produces a list of correlated variables.
   *
   * @param correlations         correlated variables to generate
   * @param valueGenFieldOffset  offset in the output that generated columns
   *                             will start
   * @param mapCorVarToOutputPos output positions for the correlated variables
   *                             generated
   * @return RelNode the root of the resultant RelNode tree
   */
private RelNode createValueGenerator(Iterable<CorRef> correlations, int valueGenFieldOffset, SortedMap<CorDef, Integer> corDefOutputs) {
    final Map<RelNode, List<Integer>> mapNewInputToOutputs = new HashMap<>();
    final Map<RelNode, Integer> mapNewInputToNewOffset = new HashMap<>();
    // Add to map all the referenced positions (relative to each input rel).
    for (CorRef corVar : correlations) {
        final int oldCorVarOffset = corVar.field;
        final RelNode oldInput = getCorRel(corVar);
        assert oldInput != null;
        final Frame frame = map.get(oldInput);
        assert frame != null;
        final RelNode newInput = frame.r;
        final List<Integer> newLocalOutputs;
        if (!mapNewInputToOutputs.containsKey(newInput)) {
            newLocalOutputs = new ArrayList<>();
        } else {
            newLocalOutputs = mapNewInputToOutputs.get(newInput);
        }
        final int newCorVarOffset = frame.oldToNewOutputs.get(oldCorVarOffset);
        // Add all unique positions referenced.
        if (!newLocalOutputs.contains(newCorVarOffset)) {
            newLocalOutputs.add(newCorVarOffset);
        }
        mapNewInputToOutputs.put(newInput, newLocalOutputs);
    }
    int offset = 0;
    // Project only the correlated fields out of each inputRel
    // and join the projectRel together.
    // To make sure the plan does not change in terms of join order,
    // join these rels based on their occurrence in cor var list which
    // is sorted.
    final Set<RelNode> joinedInputs = new HashSet<>();
    RelNode r = null;
    for (CorRef corVar : correlations) {
        final RelNode oldInput = getCorRel(corVar);
        assert oldInput != null;
        final RelNode newInput = map.get(oldInput).r;
        assert newInput != null;
        if (!joinedInputs.contains(newInput)) {
            RelNode project = RelOptUtil.createProject(newInput, mapNewInputToOutputs.get(newInput));
            RelNode distinct = RelOptUtil.createDistinctRel(project);
            RelOptCluster cluster = distinct.getCluster();
            joinedInputs.add(newInput);
            mapNewInputToNewOffset.put(newInput, offset);
            offset += distinct.getRowType().getFieldCount();
            if (r == null) {
                r = distinct;
            } else {
                r = LogicalJoin.create(r, distinct, cluster.getRexBuilder().makeLiteral(true), ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
            }
        }
    }
    // referencing correlated variables.
    for (CorRef corRef : correlations) {
        // The first input of a Correlator is always the rel defining
        // the correlated variables.
        final RelNode oldInput = getCorRel(corRef);
        assert oldInput != null;
        final Frame frame = map.get(oldInput);
        final RelNode newInput = frame.r;
        assert newInput != null;
        final List<Integer> newLocalOutputs = mapNewInputToOutputs.get(newInput);
        final int newLocalOutput = frame.oldToNewOutputs.get(corRef.field);
        // newOutput is the index of the cor var in the referenced
        // position list plus the offset of referenced position list of
        // each newInput.
        final int newOutput = newLocalOutputs.indexOf(newLocalOutput) + mapNewInputToNewOffset.get(newInput) + valueGenFieldOffset;
        corDefOutputs.put(corRef.def(), newOutput);
    }
    return r;
}

Example 38 with RelNode

use of org.apache.calcite.rel.RelNode in project hive by apache.

the class HiveRelDecorrelator method projectJoinOutputWithNullability.

/**
   * Pulls project above the join from its RHS input. Enforces nullability
   * for join output.
   *
   * @param join          Join
   * @param project       Original project as the right-hand input of the join
   * @param nullIndicatorPos Position of null indicator
   * @return the subtree with the new LogicalProject at the root
   */
private RelNode projectJoinOutputWithNullability(LogicalJoin join, LogicalProject project, int nullIndicatorPos) {
    final RelDataTypeFactory typeFactory = join.getCluster().getTypeFactory();
    final RelNode left = join.getLeft();
    final JoinRelType joinType = join.getJoinType();
    RexInputRef nullIndicator = new RexInputRef(nullIndicatorPos, typeFactory.createTypeWithNullability(join.getRowType().getFieldList().get(nullIndicatorPos).getType(), true));
    // now create the new project
    List<Pair<RexNode, String>> newProjExprs = Lists.newArrayList();
    // project everything from the LHS and then those from the original
    // projRel
    List<RelDataTypeField> leftInputFields = left.getRowType().getFieldList();
    for (int i = 0; i < leftInputFields.size(); i++) {
        newProjExprs.add(RexInputRef.of2(i, leftInputFields));
    }
    // Marked where the projected expr is coming from so that the types will
    // become nullable for the original projections which are now coming out
    // of the nullable side of the OJ.
    boolean projectPulledAboveLeftCorrelator = joinType.generatesNullsOnRight();
    for (Pair<RexNode, String> pair : project.getNamedProjects()) {
        RexNode newProjExpr = removeCorrelationExpr(pair.left, projectPulledAboveLeftCorrelator, nullIndicator);
        newProjExprs.add(Pair.of(newProjExpr, pair.right));
    }
    return RelOptUtil.createProject(join, newProjExprs, false);
}

Example 39 with RelNode

use of org.apache.calcite.rel.RelNode in project hive by apache.

the class HiveSortLimitPullUpConstantsRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final RelNode parent = call.rel(0);
    final Sort sort = call.rel(1);
    final int count = sort.getInput().getRowType().getFieldCount();
    if (count == 1) {
        // Project operator.
        return;
    }
    final RexBuilder rexBuilder = sort.getCluster().getRexBuilder();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(sort.getInput());
    if (predicates == null) {
        return;
    }
    Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
    Map<RexNode, RexNode> constants = new HashMap<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
        if (conditionsExtracted.containsKey(expr)) {
            constants.put(expr, conditionsExtracted.get(expr));
        }
    }
    // None of the expressions are constant. Nothing to do.
    if (constants.isEmpty()) {
        return;
    }
    if (count == constants.size()) {
        // At least a single item in project is required.
        constants.remove(constants.keySet().iterator().next());
    }
    // Create expressions for Project operators before and after the Sort
    List<RelDataTypeField> fields = sort.getInput().getRowType().getFieldList();
    List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
    List<RexNode> topChildExprs = new ArrayList<>();
    List<String> topChildExprsFields = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
        RelDataTypeField field = fields.get(i);
        if (constants.containsKey(expr)) {
            topChildExprs.add(constants.get(expr));
            topChildExprsFields.add(field.getName());
        } else {
            newChildExprs.add(Pair.<RexNode, String>of(expr, field.getName()));
            topChildExprs.add(expr);
            topChildExprsFields.add(field.getName());
        }
    }
    // Update field collations
    final Mappings.TargetMapping mapping = RelOptUtil.permutation(Pair.left(newChildExprs), sort.getInput().getRowType()).inverse();
    List<RelFieldCollation> fieldCollations = new ArrayList<>();
    for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
        final int target = mapping.getTargetOpt(fc.getFieldIndex());
        if (target < 0) {
            // It is a constant, we can ignore it
            continue;
        }
        fieldCollations.add(fc.copy(target));
    }
    // Update top Project positions
    topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
    // Create new Project-Sort-Project sequence
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(sort.getInput());
    relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
    final ImmutableList<RexNode> sortFields = relBuilder.fields(RelCollations.of(fieldCollations));
    relBuilder.sortLimit(sort.offset == null ? -1 : RexLiteral.intValue(sort.offset), sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch), sortFields);
    // Create top Project fixing nullability of fields
    relBuilder.project(topChildExprs, topChildExprsFields);
    relBuilder.convert(sort.getRowType(), false);
    List<RelNode> inputs = new ArrayList<>();
    for (RelNode child : parent.getInputs()) {
        if (!((HepRelVertex) child).getCurrentRel().equals(sort)) {
            inputs.add(child);
        } else {
            inputs.add(relBuilder.build());
        }
    }
    call.transformTo(parent.copy(parent.getTraitSet(), inputs));
}

Example 40 with RelNode

use of org.apache.calcite.rel.RelNode in project hive by apache.

the class HiveSortUnionReduceRule method onMatch.

public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit sort = call.rel(0);
    final HiveUnion union = call.rel(1);
    List<RelNode> inputs = new ArrayList<>();
    // Thus we use 'finishPushSortPastUnion' as a flag to identify if we have finished pushing the
    // sort past a union.
    boolean finishPushSortPastUnion = true;
    final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
    for (RelNode input : union.getInputs()) {
        // If we do not reduce the input size, we bail out
        if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.instance().getRowCount(input)) {
            finishPushSortPastUnion = false;
            // Here we do some query rewrite. We first get the new fetchRN, which is
            // a sum of offset and fetch.
            // We then push it through by creating a new branchSort with the new
            // fetchRN but no offset.
            RexNode fetchRN = sort.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(RexLiteral.intValue(sort.fetch) + offset));
            HiveSortLimit branchSort = sort.copy(sort.getTraitSet(), input, sort.getCollation(), null, fetchRN);
            branchSort.setRuleCreated(true);
            inputs.add(branchSort);
        } else {
            inputs.add(input);
        }
    }
    // there is nothing to change
    if (finishPushSortPastUnion) {
        return;
    }
    // create new union and sort
    HiveUnion unionCopy = (HiveUnion) union.copy(union.getTraitSet(), inputs, union.all);
    HiveSortLimit result = sort.copy(sort.getTraitSet(), unionCopy, sort.getCollation(), sort.offset, sort.fetch);
    call.transformTo(result);
}