Examples with HiveSortLimit org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit used on opensource projects

Example 1 with HiveSortLimit

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit in project hive by apache.

the class PlanModifierUtil method fixTopOBSchema.

protected static void fixTopOBSchema(final RelNode rootRel, Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema, boolean replaceProject) throws CalciteSemanticException {
    if (!(topSelparentPair.getKey() instanceof Sort) || !HiveCalciteUtil.orderRelNode(topSelparentPair.getKey())) {
        return;
    }
    HiveSortLimit obRel = (HiveSortLimit) topSelparentPair.getKey();
    Project obChild = (Project) topSelparentPair.getValue();
    if (obChild.getRowType().getFieldCount() <= resultSchema.size()) {
        return;
    }
    RelDataType rt = obChild.getRowType();
    @SuppressWarnings({ "unchecked", "rawtypes" }) Set<Integer> collationInputRefs = new HashSet(RelCollations.ordinals(obRel.getCollation()));
    ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
    for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
        if (collationInputRefs.contains(i)) {
            RexNode obyExpr = obChild.getProjects().get(i);
            if (obyExpr instanceof RexCall) {
                LOG.debug("Old RexCall : " + obyExpr);
                obyExpr = adjustOBSchema((RexCall) obyExpr, obChild, resultSchema);
                LOG.debug("New RexCall : " + obyExpr);
            }
            inputRefToCallMapBldr.put(i, obyExpr);
        }
    }
    ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
    if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) {
        LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size()));
        throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
    }
    if (replaceProject) {
        // This removes order-by only expressions from the projections.
        HiveProject replacementProjectRel = HiveProject.create(obChild.getInput(), obChild.getProjects().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames().subList(0, resultSchema.size()));
        obRel.replaceInput(0, replacementProjectRel);
    }
    obRel.setInputRefToCallMap(inputRefToCallMap);
}

Example 2 with HiveSortLimit

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit in project hive by apache.

the class HiveSortJoinReduceRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit sortLimit = call.rel(0);
    final HiveJoin join = call.rel(1);
    RelNode inputLeft = join.getLeft();
    RelNode inputRight = join.getRight();
    // We create a new sort operator on the corresponding input
    if (join.getJoinType() == JoinRelType.LEFT) {
        inputLeft = sortLimit.copy(sortLimit.getTraitSet(), inputLeft, sortLimit.getCollation(), sortLimit.offset, sortLimit.fetch);
        ((HiveSortLimit) inputLeft).setRuleCreated(true);
    } else {
        // Adjust right collation
        final RelCollation rightCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollations.shift(sortLimit.getCollation(), -join.getLeft().getRowType().getFieldCount()));
        inputRight = sortLimit.copy(sortLimit.getTraitSet().replace(rightCollation), inputRight, rightCollation, sortLimit.offset, sortLimit.fetch);
        ((HiveSortLimit) inputRight).setRuleCreated(true);
    }
    // We copy the join and the top sort operator
    RelNode result = join.copy(join.getTraitSet(), join.getCondition(), inputLeft, inputRight, join.getJoinType(), join.isSemiJoinDone());
    result = sortLimit.copy(sortLimit.getTraitSet(), result, sortLimit.getCollation(), sortLimit.offset, sortLimit.fetch);
    call.transformTo(result);
}

Example 3 with HiveSortLimit

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit in project hive by apache.

the class HiveSortUnionReduceRule method onMatch.

public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit sort = call.rel(0);
    final HiveUnion union = call.rel(1);
    List<RelNode> inputs = new ArrayList<>();
    // Thus we use 'finishPushSortPastUnion' as a flag to identify if we have finished pushing the
    // sort past a union.
    boolean finishPushSortPastUnion = true;
    final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
    for (RelNode input : union.getInputs()) {
        // If we do not reduce the input size, we bail out
        if (RexLiteral.intValue(sort.fetch) + offset < call.getMetadataQuery().getRowCount(input)) {
            finishPushSortPastUnion = false;
            // Here we do some query rewrite. We first get the new fetchRN, which is
            // a sum of offset and fetch.
            // We then push it through by creating a new branchSort with the new
            // fetchRN but no offset.
            RexNode fetchRN = sort.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(RexLiteral.intValue(sort.fetch) + offset));
            HiveSortLimit branchSort = sort.copy(sort.getTraitSet(), input, sort.getCollation(), null, fetchRN);
            branchSort.setRuleCreated(true);
            inputs.add(branchSort);
        } else {
            inputs.add(input);
        }
    }
    // there is nothing to change
    if (finishPushSortPastUnion) {
        return;
    }
    // create new union and sort
    HiveUnion unionCopy = (HiveUnion) union.copy(union.getTraitSet(), inputs, union.all);
    HiveSortLimit result = sort.copy(sort.getTraitSet(), unionCopy, sort.getCollation(), sort.offset, sort.fetch);
    call.transformTo(result);
}

Example 4 with HiveSortLimit

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit in project hive by apache.

the class HiveSortJoinReduceRule method matches.

// ~ Methods ----------------------------------------------------------------
@Override
public boolean matches(RelOptRuleCall call) {
    final HiveSortLimit sortLimit = call.rel(0);
    final HiveJoin join = call.rel(1);
    // If sort does not contain a limit operation or limit is 0, we bail out
    if (!HiveCalciteUtil.limitRelNode(sortLimit) || RexLiteral.intValue(sortLimit.fetch) == 0) {
        return false;
    }
    // 1) If join is not a left or right outer, we bail out
    // 2) If any sort column is not part of the input where the
    // sort is pushed, we bail out
    RelNode reducedInput;
    if (join.getJoinType() == JoinRelType.LEFT) {
        reducedInput = join.getLeft();
        if (sortLimit.getCollation() != RelCollations.EMPTY) {
            for (RelFieldCollation relFieldCollation : sortLimit.getCollation().getFieldCollations()) {
                if (relFieldCollation.getFieldIndex() >= join.getLeft().getRowType().getFieldCount()) {
                    return false;
                }
            }
        }
    } else if (join.getJoinType() == JoinRelType.RIGHT) {
        reducedInput = join.getRight();
        if (sortLimit.getCollation() != RelCollations.EMPTY) {
            for (RelFieldCollation relFieldCollation : sortLimit.getCollation().getFieldCollations()) {
                if (relFieldCollation.getFieldIndex() < join.getLeft().getRowType().getFieldCount()) {
                    return false;
                }
            }
        }
    } else {
        return false;
    }
    // Finally, if we do not reduce the input size, we bail out
    final int offset = sortLimit.offset == null ? 0 : RexLiteral.intValue(sortLimit.offset);
    final RelMetadataQuery mq = call.getMetadataQuery();
    if (offset + RexLiteral.intValue(sortLimit.fetch) >= mq.getRowCount(reducedInput)) {
        return false;
    }
    return true;
}

Example 5 with HiveSortLimit

use of org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit in project hive by apache.

the class HiveSortMergeRule method onMatch.

// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit topSortLimit = call.rel(0);
    final HiveSortLimit bottomSortLimit = call.rel(1);
    final RexNode newOffset;
    final RexNode newLimit;
    if (HiveCalciteUtil.limitRelNode(bottomSortLimit)) {
        final RexBuilder rexBuilder = topSortLimit.getCluster().getRexBuilder();
        int topOffset = topSortLimit.offset == null ? 0 : RexLiteral.intValue(topSortLimit.offset);
        int topLimit = RexLiteral.intValue(topSortLimit.fetch);
        int bottomOffset = bottomSortLimit.offset == null ? 0 : RexLiteral.intValue(bottomSortLimit.offset);
        int bottomLimit = RexLiteral.intValue(bottomSortLimit.fetch);
        // Three different cases
        if (topOffset + topLimit <= bottomLimit) {
            // 1. Fully contained
            // topOffset + topLimit <= bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = topSortLimit.fetch;
        } else if (topOffset < bottomLimit) {
            // 2. Partially contained
            // topOffset + topLimit > bottomLimit && topOffset < bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomLimit - topOffset));
        } else {
            // 3. Outside
            // we need to create a new limit 0
            newOffset = null;
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0));
        }
    } else {
        // Bottom operator does not contain offset/fetch
        newOffset = topSortLimit.offset;
        newLimit = topSortLimit.fetch;
    }
    final HiveSortLimit newSort = bottomSortLimit.copy(bottomSortLimit.getTraitSet(), bottomSortLimit.getInput(), bottomSortLimit.collation, newOffset, newLimit);
    call.transformTo(newSort);
}