Examples with RexNode org.apache.calcite.rex.RexNode used on opensource projects

Example 26 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveRelMdRowCount method canHandleJoin.

/*
   * 1. Join condition must be an Equality Predicate.
   * 2. both sides must reference 1 column.
   * 3. If needed flip the columns.
   */
private static Pair<Integer, Integer> canHandleJoin(Join joinRel, List<RexNode> leftFilters, List<RexNode> rightFilters, List<RexNode> joinFilters) {
    /*
     * If after classifying filters there is more than 1 joining predicate, we
     * don't handle this. Return null.
     */
    if (joinFilters.size() != 1) {
        return null;
    }
    RexNode joinCond = joinFilters.get(0);
    int leftColIdx;
    int rightColIdx;
    if (!(joinCond instanceof RexCall)) {
        return null;
    }
    if (((RexCall) joinCond).getOperator() != SqlStdOperatorTable.EQUALS) {
        return null;
    }
    ImmutableBitSet leftCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(0));
    ImmutableBitSet rightCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(1));
    if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1) {
        return null;
    }
    int nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size();
    int nFieldsRight = joinRel.getRight().getRowType().getFieldList().size();
    int nSysFields = joinRel.getSystemFieldList().size();
    ImmutableBitSet rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
    /*
     * flip column references if join condition specified in reverse order to
     * join sources.
     */
    if (rightFieldsBitSet.contains(leftCols)) {
        ImmutableBitSet t = leftCols;
        leftCols = rightCols;
        rightCols = t;
    }
    leftColIdx = leftCols.nextSetBit(0) - nSysFields;
    rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft);
    return new Pair<Integer, Integer>(leftColIdx, rightColIdx);
}

Example 27 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveRelMdSelectivity method computeInnerJoinSelectivity.

private Double computeInnerJoinSelectivity(Join j, RelMetadataQuery mq, RexNode predicate) {
    Pair<Boolean, RexNode> predInfo = getCombinedPredicateForJoin(j, predicate);
    if (!predInfo.getKey()) {
        return new FilterSelectivityEstimator(j).estimateSelectivity(predInfo.getValue());
    }
    RexNode combinedPredicate = predInfo.getValue();
    JoinPredicateInfo jpi;
    try {
        jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, combinedPredicate);
    } catch (CalciteSemanticException e) {
        throw new RuntimeException(e);
    }
    ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap.builder();
    ImmutableMap<Integer, Double> colStatMap;
    int rightOffSet = j.getLeft().getRowType().getFieldCount();
    // Join which are part of join keys
    for (Integer ljk : jpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) {
        colStatMapBuilder.put(ljk, HiveRelMdDistinctRowCount.getDistinctRowCount(j.getLeft(), mq, ljk));
    }
    // Join which are part of join keys
    for (Integer rjk : jpi.getProjsFromRightPartOfJoinKeysInChildSchema()) {
        colStatMapBuilder.put(rjk + rightOffSet, HiveRelMdDistinctRowCount.getDistinctRowCount(j.getRight(), mq, rjk));
    }
    colStatMap = colStatMapBuilder.build();
    // 3. Walk through the Join Condition Building NDV for selectivity
    // NDV of the join can not exceed the cardinality of cross join.
    List<JoinLeafPredicateInfo> peLst = jpi.getEquiJoinPredicateElements();
    int noOfPE = peLst.size();
    double ndvCrossProduct = 1;
    if (noOfPE > 0) {
        ndvCrossProduct = exponentialBackoff(peLst, colStatMap);
        if (j instanceof SemiJoin) {
            ndvCrossProduct = Math.min(mq.getRowCount(j.getLeft()), ndvCrossProduct);
        } else if (j instanceof HiveJoin) {
            ndvCrossProduct = Math.min(mq.getRowCount(j.getLeft()) * mq.getRowCount(j.getRight()), ndvCrossProduct);
        } else {
            throw new RuntimeException("Unexpected Join type: " + j.getClass().getName());
        }
    }
    // 4. Join Selectivity = 1/NDV
    return (1 / ndvCrossProduct);
}

Example 28 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveOpConverter method translateJoin.

private OpAttr translateJoin(RelNode joinRel) throws SemanticException {
    // 0. Additional data structures needed for the join optimization
    // through Hive
    String[] baseSrc = new String[joinRel.getInputs().size()];
    String tabAlias = getHiveDerivedTableAlias();
    // 1. Convert inputs
    OpAttr[] inputs = new OpAttr[joinRel.getInputs().size()];
    List<Operator<?>> children = new ArrayList<Operator<?>>(joinRel.getInputs().size());
    for (int i = 0; i < inputs.length; i++) {
        inputs[i] = dispatch(joinRel.getInput(i));
        children.add(inputs[i].inputs.get(0));
        baseSrc[i] = inputs[i].tabAlias;
    }
    // 2. Generate tags
    for (int tag = 0; tag < children.size(); tag++) {
        ReduceSinkOperator reduceSinkOp = (ReduceSinkOperator) children.get(tag);
        reduceSinkOp.getConf().setTag(tag);
    }
    // 3. Virtual columns
    Set<Integer> newVcolsInCalcite = new HashSet<Integer>();
    newVcolsInCalcite.addAll(inputs[0].vcolsInCalcite);
    if (joinRel instanceof HiveMultiJoin || !(joinRel instanceof SemiJoin)) {
        int shift = inputs[0].inputs.get(0).getSchema().getSignature().size();
        for (int i = 1; i < inputs.length; i++) {
            newVcolsInCalcite.addAll(HiveCalciteUtil.shiftVColsSet(inputs[i].vcolsInCalcite, shift));
            shift += inputs[i].inputs.get(0).getSchema().getSignature().size();
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + joinRel.getId() + ":" + joinRel.getRelTypeName() + " with row type: [" + joinRel.getRowType() + "]");
    }
    // 4. Extract join key expressions from HiveSortExchange
    ExprNodeDesc[][] joinExpressions = new ExprNodeDesc[inputs.length][];
    for (int i = 0; i < inputs.length; i++) {
        joinExpressions[i] = ((HiveSortExchange) joinRel.getInput(i)).getJoinExpressions();
    }
    // 5. Extract rest of join predicate info. We infer the rest of join condition
    //    that will be added to the filters (join conditions that are not part of
    //    the join key)
    List<RexNode> joinFilters;
    if (joinRel instanceof HiveJoin) {
        joinFilters = ImmutableList.of(((HiveJoin) joinRel).getJoinFilter());
    } else if (joinRel instanceof HiveMultiJoin) {
        joinFilters = ((HiveMultiJoin) joinRel).getJoinFilters();
    } else if (joinRel instanceof HiveSemiJoin) {
        joinFilters = ImmutableList.of(((HiveSemiJoin) joinRel).getJoinFilter());
    } else {
        throw new SemanticException("Can't handle join type: " + joinRel.getClass().getName());
    }
    List<List<ExprNodeDesc>> filterExpressions = Lists.newArrayList();
    for (int i = 0; i < joinFilters.size(); i++) {
        List<ExprNodeDesc> filterExpressionsForInput = new ArrayList<ExprNodeDesc>();
        if (joinFilters.get(i) != null) {
            for (RexNode conj : RelOptUtil.conjunctions(joinFilters.get(i))) {
                ExprNodeDesc expr = convertToExprNode(conj, joinRel, null, newVcolsInCalcite);
                filterExpressionsForInput.add(expr);
            }
        }
        filterExpressions.add(filterExpressionsForInput);
    }
    // 6. Generate Join operator
    JoinOperator joinOp = genJoin(joinRel, joinExpressions, filterExpressions, children, baseSrc, tabAlias);
    // 7. Return result
    return new OpAttr(tabAlias, newVcolsInCalcite, joinOp);
}

Example 29 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveOpConverter method createColInfos.

private static Pair<ArrayList<ColumnInfo>, Set<Integer>> createColInfos(List<RexNode> calciteExprs, List<ExprNodeDesc> hiveExprs, List<String> projNames, OpAttr inpOpAf) {
    if (hiveExprs.size() != projNames.size()) {
        throw new RuntimeException("Column expressions list doesn't match Column Names list");
    }
    RexNode rexN;
    ExprNodeDesc pe;
    ArrayList<ColumnInfo> colInfos = new ArrayList<ColumnInfo>();
    boolean vc;
    Set<Integer> newVColSet = new HashSet<Integer>();
    for (int i = 0; i < hiveExprs.size(); i++) {
        pe = hiveExprs.get(i);
        rexN = calciteExprs.get(i);
        vc = false;
        if (rexN instanceof RexInputRef) {
            if (inpOpAf.vcolsInCalcite.contains(((RexInputRef) rexN).getIndex())) {
                newVColSet.add(i);
                vc = true;
            }
        }
        colInfos.add(new ColumnInfo(projNames.get(i), pe.getTypeInfo(), inpOpAf.tabAlias, vc));
    }
    return new Pair<ArrayList<ColumnInfo>, Set<Integer>>(colInfos, newVColSet);
}

Example 30 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveRelColumnsAlignment method align.

public RelNode align(Project rel, List<RelFieldCollation> collations) {
    // 1) We extract the collations indices
    boolean containsWindowing = false;
    for (RexNode childExp : rel.getChildExps()) {
        if (childExp instanceof RexOver) {
            // TODO: support propagation for partitioning/ordering in windowing
            containsWindowing = true;
            break;
        }
    }
    ImmutableList.Builder<RelFieldCollation> propagateCollations = ImmutableList.builder();
    if (!containsWindowing) {
        for (RelFieldCollation c : collations) {
            RexNode rexNode = rel.getChildExps().get(c.getFieldIndex());
            if (rexNode instanceof RexInputRef) {
                int newIdx = ((RexInputRef) rexNode).getIndex();
                propagateCollations.add(c.copy((newIdx)));
            }
        }
    }
    // 2) We propagate
    final RelNode child = dispatchAlign(rel.getInput(), propagateCollations.build());
    // 3) Return new Project
    return rel.copy(rel.getTraitSet(), ImmutableList.of(child));
}