Search in sources :

Example 11 with RexFieldAccess

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess in project hive by apache.

the class HiveRelDecorrelator method checkCorVars.

/**
 * Checks whether the correlations in projRel and filter are related to
 * the correlated variables provided by corRel.
 *
 * @param correlate    Correlate
 * @param project   The original Project as the RHS input of the join
 * @param filter    Filter
 * @param correlatedJoinKeys Correlated join keys
 * @return true if filter and proj only references corVar provided by corRel
 */
private boolean checkCorVars(LogicalCorrelate correlate, LogicalProject project, LogicalFilter filter, List<RexFieldAccess> correlatedJoinKeys) {
    if (filter != null) {
        assert correlatedJoinKeys != null;
        // check that all correlated refs in the filter condition are
        // used in the join(as field access).
        Set<CorRef> corVarInFilter = Sets.newHashSet(cm.mapRefRelToCorRef.get(filter));
        for (RexFieldAccess correlatedJoinKey : correlatedJoinKeys) {
            corVarInFilter.remove(cm.mapFieldAccessToCorRef.get(correlatedJoinKey));
        }
        if (!corVarInFilter.isEmpty()) {
            return false;
        }
        // Check that the correlated variables referenced in these
        // comparisons do come from the correlatorRel.
        corVarInFilter.addAll(cm.mapRefRelToCorRef.get(filter));
        for (CorRef corVar : corVarInFilter) {
            if (cm.mapCorToCorRel.get(corVar.corr) != correlate) {
                return false;
            }
        }
    }
    // of the correlate.
    if ((project != null) && cm.mapRefRelToCorRef.containsKey(project)) {
        for (CorRef corVar : cm.mapRefRelToCorRef.get(project)) {
            if (cm.mapCorToCorRel.get(corVar.corr) != correlate) {
                return false;
            }
        }
    }
    return true;
}
Also used : RexFieldAccess(org.apache.calcite.rex.RexFieldAccess)

Example 12 with RexFieldAccess

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess in project hive by apache.

the class HiveRelDecorrelator method checkCorVars.

/**
 * Checks whether the correlations in projRel and filter are related to
 * the correlated variables provided by corRel.
 *
 * @param correlate    Correlate
 * @param project   The original Project as the RHS input of the join
 * @param filter    Filter
 * @param correlatedJoinKeys Correlated join keys
 * @return true if filter and proj only references corVar provided by corRel
 */
private boolean checkCorVars(LogicalCorrelate correlate, Project project, Filter filter, List<RexFieldAccess> correlatedJoinKeys) {
    if (filter != null) {
        assert correlatedJoinKeys != null;
        // check that all correlated refs in the filter condition are
        // used in the join(as field access).
        Set<CorRef> corVarInFilter = Sets.newHashSet(cm.mapRefRelToCorRef.get(filter));
        for (RexFieldAccess correlatedJoinKey : correlatedJoinKeys) {
            corVarInFilter.remove(cm.mapFieldAccessToCorRef.get(correlatedJoinKey));
        }
        if (!corVarInFilter.isEmpty()) {
            return false;
        }
        // Check that the correlated variables referenced in these
        // comparisons do come from the correlatorRel.
        corVarInFilter.addAll(cm.mapRefRelToCorRef.get(filter));
        for (CorRef corVar : corVarInFilter) {
            if (cm.mapCorToCorRel.get(corVar.corr) != correlate) {
                return false;
            }
        }
    }
    // of the correlate.
    if ((project != null) && cm.mapRefRelToCorRef.containsKey(project)) {
        for (CorRef corVar : cm.mapRefRelToCorRef.get(project)) {
            if (cm.mapCorToCorRel.get(corVar.corr) != correlate) {
                return false;
            }
        }
    }
    return true;
}
Also used : RexFieldAccess(org.apache.calcite.rex.RexFieldAccess)

Example 13 with RexFieldAccess

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess in project hive by apache.

the class CalcitePlanner method genAllRexNode.

/**
 * Generates all of the Calcite {@link RexNode}s for the expression and children of it
 * passed in the arguments. This function uses the row resolver and the metadata information
 * that are passed as arguments to resolve the column names to internal names.
 *
 * @param expr
 *          The expression
 * @param input
 *          The row resolver
 * @param tcCtx
 *          Customized type-checking context
 * @return expression to exprNodeDesc mapping
 * @throws SemanticException Failed to evaluate expression
 */
Map<ASTNode, RexNode> genAllRexNode(ASTNode expr, RowResolver input, TypeCheckCtx tcCtx) throws SemanticException {
    // Create the walker and  the rules dispatcher.
    tcCtx.setUnparseTranslator(unparseTranslator);
    Map<ASTNode, RexNode> nodeOutputs = RexNodeTypeCheck.genExprNode(expr, tcCtx);
    RexNode desc = nodeOutputs.get(expr);
    if (desc == null) {
        String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
        ColumnInfo colInfo = input.get(null, tableOrCol);
        String errMsg;
        if (colInfo == null && input.getIsExprResolver()) {
            errMsg = ASTErrorUtils.getMsg(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(), expr);
        } else {
            errMsg = tcCtx.getError();
        }
        throw new SemanticException(Optional.ofNullable(errMsg).orElse("Error in parsing "));
    }
    if (desc instanceof HiveRexExprList) {
        throw new SemanticException("TOK_ALLCOLREF is not supported in current context");
    }
    if (!unparseTranslator.isEnabled()) {
        // Not creating a view, so no need to track view expansions.
        return nodeOutputs;
    }
    List<ASTNode> fieldDescList = new ArrayList<>();
    for (Map.Entry<ASTNode, RexNode> entry : nodeOutputs.entrySet()) {
        if (!(entry.getValue() instanceof RexInputRef)) {
            // struct<>.
            if (entry.getValue() instanceof RexFieldAccess) {
                fieldDescList.add(entry.getKey());
            }
            continue;
        }
        ASTNode node = entry.getKey();
        RexInputRef columnDesc = (RexInputRef) entry.getValue();
        int index = columnDesc.getIndex();
        String[] tmp;
        if (index < input.getColumnInfos().size()) {
            ColumnInfo columnInfo = input.getColumnInfos().get(index);
            if (columnInfo.getTabAlias() == null || columnInfo.getTabAlias().length() == 0) {
                // internal expressions used in the representation of aggregation.
                continue;
            }
            tmp = input.reverseLookup(columnInfo.getInternalName());
        } else {
            // in subquery case, tmp may be from outside.
            ColumnInfo columnInfo = tcCtx.getOuterRR().getColumnInfos().get(index - input.getColumnInfos().size());
            if (columnInfo.getTabAlias() == null || columnInfo.getTabAlias().length() == 0) {
                continue;
            }
            tmp = tcCtx.getOuterRR().reverseLookup(columnInfo.getInternalName());
        }
        StringBuilder replacementText = new StringBuilder();
        replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf));
        replacementText.append(".");
        replacementText.append(HiveUtils.unparseIdentifier(tmp[1], conf));
        unparseTranslator.addTranslation(node, replacementText.toString());
    }
    for (ASTNode node : fieldDescList) {
        Map<ASTNode, String> map = translateFieldDesc(node);
        for (Entry<ASTNode, String> entry : map.entrySet()) {
            unparseTranslator.addTranslation(entry.getKey(), entry.getValue().toLowerCase());
        }
    }
    return nodeOutputs;
}
Also used : ArrayList(java.util.ArrayList) ColumnInfo(org.apache.hadoop.hive.ql.exec.ColumnInfo) NotNullConstraint(org.apache.hadoop.hive.ql.metadata.NotNullConstraint) HiveRexExprList(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList) RexInputRef(org.apache.calcite.rex.RexInputRef) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) RexFieldAccess(org.apache.calcite.rex.RexFieldAccess) RexNode(org.apache.calcite.rex.RexNode) CalciteSubquerySemanticException(org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException) CalciteSemanticException(org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException) CalciteViewSemanticException(org.apache.hadoop.hive.ql.optimizer.calcite.CalciteViewSemanticException)

Example 14 with RexFieldAccess

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess in project hive by apache.

the class CorrelateProjectExtractor method visit.

@Override
public RelNode visit(LogicalCorrelate correlate) {
    RelNode left = correlate.getLeft().accept(this);
    RelNode right = correlate.getRight().accept(this);
    int oldLeft = left.getRowType().getFieldCount();
    // Find the correlated expressions from the right side that can be moved to the left
    Set<RexNode> callsWithCorrelationInRight = findCorrelationDependentCalls(correlate.getCorrelationId(), right);
    boolean isTrivialCorrelation = callsWithCorrelationInRight.stream().allMatch(exp -> exp instanceof RexFieldAccess);
    // Early exit condition
    if (isTrivialCorrelation) {
        if (correlate.getLeft().equals(left) && correlate.getRight().equals(right)) {
            return correlate;
        } else {
            return correlate.copy(correlate.getTraitSet(), left, right, correlate.getCorrelationId(), correlate.getRequiredColumns(), correlate.getJoinType());
        }
    }
    RelBuilder builder = builderFactory.create(correlate.getCluster(), null);
    // Transform the correlated expression from the right side to an expression over the left side
    builder.push(left);
    List<RexNode> callsWithCorrelationOverLeft = new ArrayList<>();
    for (RexNode callInRight : callsWithCorrelationInRight) {
        callsWithCorrelationOverLeft.add(replaceCorrelationsWithInputRef(callInRight, builder));
    }
    builder.projectPlus(callsWithCorrelationOverLeft);
    // Construct the mapping to transform the expressions in the right side based on the new
    // projection in the left side.
    Map<RexNode, RexNode> transformMapping = new HashMap<>();
    for (RexNode callInRight : callsWithCorrelationInRight) {
        RexBuilder xb = builder.getRexBuilder();
        RexNode v = xb.makeCorrel(builder.peek().getRowType(), correlate.getCorrelationId());
        RexNode flatCorrelationInRight = xb.makeFieldAccess(v, oldLeft + transformMapping.size());
        transformMapping.put(callInRight, flatCorrelationInRight);
    }
    // Select the required fields/columns from the left side of the correlation. Based on the code
    // above all these fields should be at the end of the left relational expression.
    List<RexNode> requiredFields = builder.fields(ImmutableBitSet.range(oldLeft, oldLeft + callsWithCorrelationOverLeft.size()).asList());
    final int newLeft = builder.fields().size();
    // Transform the expressions in the right side using the mapping constructed earlier.
    right = replaceExpressionsUsingMap(right, transformMapping);
    builder.push(right);
    builder.correlate(correlate.getJoinType(), correlate.getCorrelationId(), requiredFields);
    // Remove the additional fields that were added for the needs of the correlation to keep the old
    // and new plan equivalent.
    List<Integer> retainFields;
    switch(correlate.getJoinType()) {
        case SEMI:
        case ANTI:
            retainFields = ImmutableBitSet.range(0, oldLeft).asList();
            break;
        case LEFT:
        case INNER:
            retainFields = ImmutableBitSet.builder().set(0, oldLeft).set(newLeft, newLeft + right.getRowType().getFieldCount()).build().asList();
            break;
        default:
            throw new AssertionError(correlate.getJoinType());
    }
    builder.project(builder.fields(retainFields));
    return builder.build();
}
Also used : RelBuilder(org.apache.calcite.tools.RelBuilder) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) RelNode(org.apache.calcite.rel.RelNode) RexBuilder(org.apache.calcite.rex.RexBuilder) RexFieldAccess(org.apache.calcite.rex.RexFieldAccess) RexNode(org.apache.calcite.rex.RexNode)

Example 15 with RexFieldAccess

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess in project beam by apache.

the class BeamUnnestRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    LogicalCorrelate correlate = call.rel(0);
    RelNode outer = call.rel(1);
    RelNode uncollect = call.rel(2);
    if (correlate.getCorrelationId().getId() != 0) {
        // Only one level of correlation nesting is supported
        return;
    }
    if (correlate.getRequiredColumns().cardinality() != 1) {
        // can only unnest a single column
        return;
    }
    if (correlate.getJoinType() != JoinRelType.INNER) {
        return;
    }
    if (!(uncollect instanceof Uncollect)) {
        // Drop projection
        uncollect = ((SingleRel) uncollect).getInput();
        if (uncollect instanceof RelSubset) {
            uncollect = ((RelSubset) uncollect).getOriginal();
        }
        if (!(uncollect instanceof Uncollect)) {
            return;
        }
    }
    RelNode project = ((Uncollect) uncollect).getInput();
    if (project instanceof RelSubset) {
        project = ((RelSubset) project).getOriginal();
    }
    if (!(project instanceof LogicalProject)) {
        return;
    }
    if (((LogicalProject) project).getProjects().size() != 1) {
        // can only unnest a single column
        return;
    }
    RexNode exp = ((LogicalProject) project).getProjects().get(0);
    if (!(exp instanceof RexFieldAccess)) {
        return;
    }
    RexFieldAccess fieldAccess = (RexFieldAccess) exp;
    // Innermost field index comes first (e.g. struct.field1.field2 => [2, 1])
    ImmutableList.Builder<Integer> fieldAccessIndices = ImmutableList.builder();
    while (true) {
        fieldAccessIndices.add(fieldAccess.getField().getIndex());
        if (!(fieldAccess.getReferenceExpr() instanceof RexFieldAccess)) {
            break;
        }
        fieldAccess = (RexFieldAccess) fieldAccess.getReferenceExpr();
    }
    call.transformTo(new BeamUnnestRel(correlate.getCluster(), correlate.getTraitSet().replace(BeamLogicalConvention.INSTANCE), convert(outer, outer.getTraitSet().replace(BeamLogicalConvention.INSTANCE)), call.rel(2).getRowType(), fieldAccessIndices.build()));
}
Also used : Uncollect(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Uncollect) RelNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode) ImmutableList(org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableList) LogicalCorrelate(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalCorrelate) LogicalProject(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject) RexFieldAccess(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess) RelSubset(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.volcano.RelSubset) RexNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode) BeamUnnestRel(org.apache.beam.sdk.extensions.sql.impl.rel.BeamUnnestRel)

Aggregations

RexFieldAccess (org.apache.calcite.rex.RexFieldAccess)21 RexNode (org.apache.calcite.rex.RexNode)15 RexCorrelVariable (org.apache.calcite.rex.RexCorrelVariable)9 RexInputRef (org.apache.calcite.rex.RexInputRef)8 RexCall (org.apache.calcite.rex.RexCall)7 CorrelationId (org.apache.calcite.rel.core.CorrelationId)5 ArrayList (java.util.ArrayList)4 RexBuilder (org.apache.calcite.rex.RexBuilder)4 HashMap (java.util.HashMap)3 Map (java.util.Map)3 RelNode (org.apache.calcite.rel.RelNode)3 NlsString (org.apache.calcite.util.NlsString)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 List (java.util.List)2 RelSubset (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.volcano.RelSubset)2 RelNode (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode)2 LogicalCorrelate (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalCorrelate)2 LogicalProject (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.logical.LogicalProject)2 RexFieldAccess (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexFieldAccess)2 RexNode (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode)2