Examples with RelFieldCollation org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation used on opensource projects

Example 26 with RelFieldCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation in project drill by apache.

the class DrillSortRel method implement.

@Override
public LogicalOperator implement(DrillImplementor implementor) {
    final Order.Builder builder = Order.builder();
    builder.setInput(implementor.visitChild(this, 0, getInput()));
    final List<String> childFields = getInput().getRowType().getFieldNames();
    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        builder.addOrdering(fieldCollation.getDirection(), new FieldReference(childFields.get(fieldCollation.getFieldIndex())), fieldCollation.nullDirection);
    }
    return builder.build();
}

Example 27 with RelFieldCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation in project drill by apache.

the class IndexPlanUtils method buildCollationLowerProject.

/**
 * Build collation property for the 'lower' project, the one closer to the Scan
 * @param projectRexs list of row expressions
 * @param input input as a relational expression
 * @param indexInfo collects functional index information
 * @return the output RelCollation
 */
public static RelCollation buildCollationLowerProject(List<RexNode> projectRexs, RelNode input, FunctionalIndexInfo indexInfo) {
    // if leading fields of index are here, add them to RelCollation
    List<RelFieldCollation> newFields = Lists.newArrayList();
    if (!indexInfo.hasFunctional()) {
        Map<LogicalExpression, Integer> projectExprs = Maps.newLinkedHashMap();
        DrillParseContext parserContext = new DrillParseContext(PrelUtil.getPlannerSettings(input.getCluster()));
        int idx = 0;
        for (RexNode rex : projectRexs) {
            projectExprs.put(DrillOptiq.toDrill(parserContext, input, rex), idx);
            idx++;
        }
        int idxFieldCount = 0;
        for (LogicalExpression expr : indexInfo.getIndexDesc().getIndexColumns()) {
            if (!projectExprs.containsKey(expr)) {
                break;
            }
            RelFieldCollation.Direction dir = indexInfo.getIndexDesc().getCollation().getFieldCollations().get(idxFieldCount).direction;
            if (dir == null) {
                break;
            }
            newFields.add(new RelFieldCollation(projectExprs.get(expr), dir, RelFieldCollation.NullDirection.UNSPECIFIED));
        }
        idxFieldCount++;
    } else {
    // TODO: handle functional index
    }
    return RelCollations.of(newFields);
}

Example 28 with RelFieldCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation in project drill by apache.

the class MetadataAggPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    MetadataAggRel aggregate = call.rel(0);
    RelNode input = aggregate.getInput();
    int groupByExprsSize = aggregate.getContext().groupByExpressions().size();
    List<RelFieldCollation> collations = new ArrayList<>();
    List<String> names = new ArrayList<>();
    for (int i = 0; i < groupByExprsSize; i++) {
        collations.add(new RelFieldCollation(i + 1));
        SchemaPath fieldPath = getArgumentReference(aggregate.getContext().groupByExpressions().get(i));
        names.add(fieldPath.getRootSegmentPath());
    }
    RelCollation collation = new NamedRelCollation(collations, names);
    RelTraitSet traits;
    if (aggregate.getContext().groupByExpressions().isEmpty()) {
        DrillDistributionTrait singleDist = DrillDistributionTrait.SINGLETON;
        RelTraitSet singleDistTrait = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(singleDist);
        createTransformRequest(call, aggregate, input, singleDistTrait);
    } else {
        // hash distribute on all grouping keys
        DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(aggregate.getContext().groupByExpressions())));
        PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
        boolean smallInput = input.estimateRowCount(input.getCluster().getMetadataQuery()) < settings.getSliceTarget();
        // to produce sort locally before aggregation is produced for large inputs
        if (aggregate.getContext().createNewAggregations() && !smallInput) {
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
            RelNode convertedInput = convert(input, traits);
            new TwoPhaseMetadataAggSubsetTransformer(call, collation, distOnAllKeys).go(aggregate, convertedInput);
        } else {
            // TODO: DRILL-7433 - replace DrillDistributionTrait.SINGLETON with distOnAllKeys when parallelization for MetadataHandler is implemented
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(DrillDistributionTrait.SINGLETON);
            createTransformRequest(call, aggregate, input, traits);
        }
    }
}

Example 29 with RelFieldCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation in project drill by apache.

the class PrelUtil method getOrdering.

public static List<Ordering> getOrdering(RelCollation collation, RelDataType rowType) {
    List<Ordering> orderExpr = new ArrayList<>();
    List<String> childFields = rowType.getFieldNames();
    Function<RelFieldCollation, String> fieldNameProvider;
    if (collation instanceof MetadataAggPrule.NamedRelCollation) {
        fieldNameProvider = fieldCollation -> {
            MetadataAggPrule.NamedRelCollation namedCollation = (MetadataAggPrule.NamedRelCollation) collation;
            return namedCollation.getName(fieldCollation.getFieldIndex());
        };
    } else {
        fieldNameProvider = fieldCollation -> childFields.get(fieldCollation.getFieldIndex());
    }
    collation.getFieldCollations().forEach(fieldCollation -> {
        FieldReference fieldReference = new FieldReference(fieldNameProvider.apply(fieldCollation), ExpressionPosition.UNKNOWN);
        orderExpr.add(new Ordering(fieldCollation.getDirection(), fieldReference, fieldCollation.nullDirection));
    });
    return orderExpr;
}

Example 30 with RelFieldCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation in project flink by apache.

the class HiveParserCalcitePlanner method genOBLogicalPlan.

private Pair<Sort, RelNode> genOBLogicalPlan(HiveParserQB qb, RelNode srcRel, boolean outermostOB) throws SemanticException {
    Sort sortRel = null;
    RelNode originalOBInput = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    String dest = qbp.getClauseNames().iterator().next();
    HiveParserASTNode obAST = qbp.getOrderByForClause(dest);
    if (obAST != null) {
        // 1. OB Expr sanity test
        // in strict mode, in the presence of order by, limit must be specified
        Integer limit = qb.getParseInfo().getDestLimit(dest);
        if (limit == null) {
            String mapRedMode = semanticAnalyzer.getConf().getVar(HiveConf.ConfVars.HIVEMAPREDMODE);
            boolean banLargeQuery = Boolean.parseBoolean(semanticAnalyzer.getConf().get("hive.strict.checks.large.query", "false"));
            if ("strict".equalsIgnoreCase(mapRedMode) || banLargeQuery) {
                throw new SemanticException(generateErrorMessage(obAST, "Order by-s without limit"));
            }
        }
        // 2. Walk through OB exprs and extract field collations and additional
        // virtual columns needed
        final List<RexNode> virtualCols = new ArrayList<>();
        final List<RelFieldCollation> fieldCollations = new ArrayList<>();
        int fieldIndex;
        List<Node> obASTExprLst = obAST.getChildren();
        HiveParserASTNode obASTExpr;
        HiveParserASTNode nullOrderASTExpr;
        List<Pair<HiveParserASTNode, TypeInfo>> vcASTAndType = new ArrayList<>();
        HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        HiveParserRexNodeConverter converter = new HiveParserRexNodeConverter(cluster, srcRel.getRowType(), relToHiveColNameCalcitePosMap.get(srcRel), 0, false, funcConverter);
        int numSrcFields = srcRel.getRowType().getFieldCount();
        for (Node node : obASTExprLst) {
            // 2.1 Convert AST Expr to ExprNode
            obASTExpr = (HiveParserASTNode) node;
            nullOrderASTExpr = (HiveParserASTNode) obASTExpr.getChild(0);
            HiveParserASTNode ref = (HiveParserASTNode) nullOrderASTExpr.getChild(0);
            Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(ref, inputRR);
            ExprNodeDesc obExprNodeDesc = astToExprNodeDesc.get(ref);
            if (obExprNodeDesc == null) {
                throw new SemanticException("Invalid order by expression: " + obASTExpr.toString());
            }
            // 2.2 Convert ExprNode to RexNode
            RexNode rexNode = converter.convert(obExprNodeDesc).accept(funcConverter);
            // present in the child (& hence we add a child Project Rel)
            if (rexNode instanceof RexInputRef) {
                fieldIndex = ((RexInputRef) rexNode).getIndex();
            } else {
                fieldIndex = numSrcFields + virtualCols.size();
                virtualCols.add(rexNode);
                vcASTAndType.add(new Pair<>(ref, obExprNodeDesc.getTypeInfo()));
            }
            // 2.4 Determine the Direction of order by
            RelFieldCollation.Direction direction = RelFieldCollation.Direction.DESCENDING;
            if (obASTExpr.getType() == HiveASTParser.TOK_TABSORTCOLNAMEASC) {
                direction = RelFieldCollation.Direction.ASCENDING;
            }
            RelFieldCollation.NullDirection nullOrder;
            if (nullOrderASTExpr.getType() == HiveASTParser.TOK_NULLS_FIRST) {
                nullOrder = RelFieldCollation.NullDirection.FIRST;
            } else if (nullOrderASTExpr.getType() == HiveASTParser.TOK_NULLS_LAST) {
                nullOrder = RelFieldCollation.NullDirection.LAST;
            } else {
                throw new SemanticException("Unexpected null ordering option: " + nullOrderASTExpr.getType());
            }
            // 2.5 Add to field collations
            fieldCollations.add(new RelFieldCollation(fieldIndex, direction, nullOrder));
        }
        // 3. Add Child Project Rel if needed, Generate Output RR, input Sel Rel
        // for top constraining Sel
        RelNode obInputRel = srcRel;
        if (!virtualCols.isEmpty()) {
            List<RexNode> originalInputRefs = srcRel.getRowType().getFieldList().stream().map(input -> new RexInputRef(input.getIndex(), input.getType())).collect(Collectors.toList());
            HiveParserRowResolver obSyntheticProjectRR = new HiveParserRowResolver();
            if (!HiveParserRowResolver.add(obSyntheticProjectRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
            int vcolPos = inputRR.getRowSchema().getSignature().size();
            for (Pair<HiveParserASTNode, TypeInfo> astTypePair : vcASTAndType) {
                obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo(getColumnInternalName(vcolPos), astTypePair.getValue(), null, false));
                vcolPos++;
            }
            obInputRel = genSelectRelNode(CompositeList.of(originalInputRefs, virtualCols), obSyntheticProjectRR, srcRel);
            if (outermostOB) {
                if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            } else {
                if (!HiveParserRowResolver.add(outputRR, obSyntheticProjectRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            }
            originalOBInput = srcRel;
        } else {
            if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
        }
        // 4. Construct SortRel
        RelTraitSet traitSet = cluster.traitSet();
        RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(fieldCollations));
        sortRel = LogicalSort.create(obInputRel, canonizedCollation, null, null);
        // 5. Update the maps
        Map<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(outputRR);
        relToRowResolver.put(sortRel, outputRR);
        relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
    }
    return (new Pair<>(sortRel, originalOBInput));
}