Search in sources :

Example 16 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project drill by apache.

the class MetadataAggPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    MetadataAggRel aggregate = call.rel(0);
    RelNode input = aggregate.getInput();
    int groupByExprsSize = aggregate.getContext().groupByExpressions().size();
    List<RelFieldCollation> collations = new ArrayList<>();
    List<String> names = new ArrayList<>();
    for (int i = 0; i < groupByExprsSize; i++) {
        collations.add(new RelFieldCollation(i + 1));
        SchemaPath fieldPath = getArgumentReference(aggregate.getContext().groupByExpressions().get(i));
        names.add(fieldPath.getRootSegmentPath());
    }
    RelCollation collation = new NamedRelCollation(collations, names);
    RelTraitSet traits;
    if (aggregate.getContext().groupByExpressions().isEmpty()) {
        DrillDistributionTrait singleDist = DrillDistributionTrait.SINGLETON;
        RelTraitSet singleDistTrait = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(singleDist);
        createTransformRequest(call, aggregate, input, singleDistTrait);
    } else {
        // hash distribute on all grouping keys
        DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(aggregate.getContext().groupByExpressions())));
        PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
        boolean smallInput = input.estimateRowCount(input.getCluster().getMetadataQuery()) < settings.getSliceTarget();
        // to produce sort locally before aggregation is produced for large inputs
        if (aggregate.getContext().createNewAggregations() && !smallInput) {
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
            RelNode convertedInput = convert(input, traits);
            new TwoPhaseMetadataAggSubsetTransformer(call, collation, distOnAllKeys).go(aggregate, convertedInput);
        } else {
            // TODO: DRILL-7433 - replace DrillDistributionTrait.SINGLETON with distOnAllKeys when parallelization for MetadataHandler is implemented
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(DrillDistributionTrait.SINGLETON);
            createTransformRequest(call, aggregate, input, traits);
        }
    }
}
Also used : ArrayList(java.util.ArrayList) RelTraitSet(org.apache.calcite.plan.RelTraitSet) MetadataAggRel(org.apache.drill.exec.planner.logical.MetadataAggRel) RelCollation(org.apache.calcite.rel.RelCollation) RelNode(org.apache.calcite.rel.RelNode) SchemaPath(org.apache.drill.common.expression.SchemaPath) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation)

Example 17 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project flink by apache.

the class HiveParserCalcitePlanner method genLimitLogicalPlan.

private Sort genLimitLogicalPlan(HiveParserQB qb, RelNode srcRel) throws SemanticException {
    Sort sortRel = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    AbstractMap.SimpleEntry<Integer, Integer> entry = qbp.getDestToLimit().get(qbp.getClauseNames().iterator().next());
    int offset = (entry == null) ? 0 : entry.getKey();
    Integer fetch = (entry == null) ? null : entry.getValue();
    if (fetch != null) {
        RexNode offsetRex = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(offset));
        RexNode fetchRex = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(fetch));
        RelTraitSet traitSet = cluster.traitSet();
        RelCollation canonizedCollation = traitSet.canonize(RelCollations.EMPTY);
        sortRel = LogicalSort.create(srcRel, canonizedCollation, offsetRex, fetchRex);
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        if (!HiveParserRowResolver.add(outputRR, relToRowResolver.get(srcRel))) {
            throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
        }
        Map<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(outputRR);
        relToRowResolver.put(sortRel, outputRR);
        relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
    }
    return sortRel;
}
Also used : RelTraitSet(org.apache.calcite.plan.RelTraitSet) AbstractMap(java.util.AbstractMap) RelCollation(org.apache.calcite.rel.RelCollation) HiveParserQBParseInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQBParseInfo) HiveParserRowResolver(org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver) LogicalSort(org.apache.calcite.rel.logical.LogicalSort) Sort(org.apache.calcite.rel.core.Sort) RexNode(org.apache.calcite.rex.RexNode) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException)

Example 18 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project flink by apache.

the class HiveParserCalcitePlanner method genOBLogicalPlan.

private Pair<Sort, RelNode> genOBLogicalPlan(HiveParserQB qb, RelNode srcRel, boolean outermostOB) throws SemanticException {
    Sort sortRel = null;
    RelNode originalOBInput = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    String dest = qbp.getClauseNames().iterator().next();
    HiveParserASTNode obAST = qbp.getOrderByForClause(dest);
    if (obAST != null) {
        // 1. OB Expr sanity test
        // in strict mode, in the presence of order by, limit must be specified
        Integer limit = qb.getParseInfo().getDestLimit(dest);
        if (limit == null) {
            String mapRedMode = semanticAnalyzer.getConf().getVar(HiveConf.ConfVars.HIVEMAPREDMODE);
            boolean banLargeQuery = Boolean.parseBoolean(semanticAnalyzer.getConf().get("hive.strict.checks.large.query", "false"));
            if ("strict".equalsIgnoreCase(mapRedMode) || banLargeQuery) {
                throw new SemanticException(generateErrorMessage(obAST, "Order by-s without limit"));
            }
        }
        // 2. Walk through OB exprs and extract field collations and additional
        // virtual columns needed
        final List<RexNode> virtualCols = new ArrayList<>();
        final List<RelFieldCollation> fieldCollations = new ArrayList<>();
        int fieldIndex;
        List<Node> obASTExprLst = obAST.getChildren();
        HiveParserASTNode obASTExpr;
        HiveParserASTNode nullOrderASTExpr;
        List<Pair<HiveParserASTNode, TypeInfo>> vcASTAndType = new ArrayList<>();
        HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        HiveParserRexNodeConverter converter = new HiveParserRexNodeConverter(cluster, srcRel.getRowType(), relToHiveColNameCalcitePosMap.get(srcRel), 0, false, funcConverter);
        int numSrcFields = srcRel.getRowType().getFieldCount();
        for (Node node : obASTExprLst) {
            // 2.1 Convert AST Expr to ExprNode
            obASTExpr = (HiveParserASTNode) node;
            nullOrderASTExpr = (HiveParserASTNode) obASTExpr.getChild(0);
            HiveParserASTNode ref = (HiveParserASTNode) nullOrderASTExpr.getChild(0);
            Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(ref, inputRR);
            ExprNodeDesc obExprNodeDesc = astToExprNodeDesc.get(ref);
            if (obExprNodeDesc == null) {
                throw new SemanticException("Invalid order by expression: " + obASTExpr.toString());
            }
            // 2.2 Convert ExprNode to RexNode
            RexNode rexNode = converter.convert(obExprNodeDesc).accept(funcConverter);
            // present in the child (& hence we add a child Project Rel)
            if (rexNode instanceof RexInputRef) {
                fieldIndex = ((RexInputRef) rexNode).getIndex();
            } else {
                fieldIndex = numSrcFields + virtualCols.size();
                virtualCols.add(rexNode);
                vcASTAndType.add(new Pair<>(ref, obExprNodeDesc.getTypeInfo()));
            }
            // 2.4 Determine the Direction of order by
            RelFieldCollation.Direction direction = RelFieldCollation.Direction.DESCENDING;
            if (obASTExpr.getType() == HiveASTParser.TOK_TABSORTCOLNAMEASC) {
                direction = RelFieldCollation.Direction.ASCENDING;
            }
            RelFieldCollation.NullDirection nullOrder;
            if (nullOrderASTExpr.getType() == HiveASTParser.TOK_NULLS_FIRST) {
                nullOrder = RelFieldCollation.NullDirection.FIRST;
            } else if (nullOrderASTExpr.getType() == HiveASTParser.TOK_NULLS_LAST) {
                nullOrder = RelFieldCollation.NullDirection.LAST;
            } else {
                throw new SemanticException("Unexpected null ordering option: " + nullOrderASTExpr.getType());
            }
            // 2.5 Add to field collations
            fieldCollations.add(new RelFieldCollation(fieldIndex, direction, nullOrder));
        }
        // 3. Add Child Project Rel if needed, Generate Output RR, input Sel Rel
        // for top constraining Sel
        RelNode obInputRel = srcRel;
        if (!virtualCols.isEmpty()) {
            List<RexNode> originalInputRefs = srcRel.getRowType().getFieldList().stream().map(input -> new RexInputRef(input.getIndex(), input.getType())).collect(Collectors.toList());
            HiveParserRowResolver obSyntheticProjectRR = new HiveParserRowResolver();
            if (!HiveParserRowResolver.add(obSyntheticProjectRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
            int vcolPos = inputRR.getRowSchema().getSignature().size();
            for (Pair<HiveParserASTNode, TypeInfo> astTypePair : vcASTAndType) {
                obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo(getColumnInternalName(vcolPos), astTypePair.getValue(), null, false));
                vcolPos++;
            }
            obInputRel = genSelectRelNode(CompositeList.of(originalInputRefs, virtualCols), obSyntheticProjectRR, srcRel);
            if (outermostOB) {
                if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            } else {
                if (!HiveParserRowResolver.add(outputRR, obSyntheticProjectRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            }
            originalOBInput = srcRel;
        } else {
            if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
        }
        // 4. Construct SortRel
        RelTraitSet traitSet = cluster.traitSet();
        RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(fieldCollations));
        sortRel = LogicalSort.create(obInputRel, canonizedCollation, null, null);
        // 5. Update the maps
        Map<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(outputRR);
        relToRowResolver.put(sortRel, outputRR);
        relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
    }
    return (new Pair<>(sortRel, originalOBInput));
}
Also used : DataType(org.apache.flink.table.types.DataType) Arrays(java.util.Arrays) LogicalSort(org.apache.calcite.rel.logical.LogicalSort) FlinkPlannerImpl(org.apache.flink.table.planner.calcite.FlinkPlannerImpl) JoinType(org.apache.hadoop.hive.ql.parse.JoinType) HiveParserBaseSemanticAnalyzer.getGroupByForClause(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getGroupByForClause) HiveParserContext(org.apache.flink.table.planner.delegation.hive.copy.HiveParserContext) FunctionRegistry(org.apache.hadoop.hive.ql.exec.FunctionRegistry) RelCollationImpl(org.apache.calcite.rel.RelCollationImpl) BigDecimal(java.math.BigDecimal) HiveParserBaseSemanticAnalyzer.unescapeIdentifier(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.unescapeIdentifier) HiveInspectors(org.apache.flink.table.functions.hive.conversion.HiveInspectors) CorrelationId(org.apache.calcite.rel.core.CorrelationId) HiveParserASTNode(org.apache.flink.table.planner.delegation.hive.copy.HiveParserASTNode) Map(java.util.Map) HiveParserASTBuilder(org.apache.flink.table.planner.delegation.hive.copy.HiveParserASTBuilder) RelTraitSet(org.apache.calcite.plan.RelTraitSet) RexWindowBound(org.apache.calcite.rex.RexWindowBound) ImmutableBitSet(org.apache.calcite.util.ImmutableBitSet) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) HiveParserSubQueryUtils(org.apache.flink.table.planner.delegation.hive.copy.HiveParserSubQueryUtils) HiveParserBaseSemanticAnalyzer.getHiveAggInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getHiveAggInfo) HiveTypeUtil(org.apache.flink.table.catalog.hive.util.HiveTypeUtil) HiveParserBaseSemanticAnalyzer(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer) org.apache.hadoop.hive.serde.serdeConstants(org.apache.hadoop.hive.serde.serdeConstants) Set(java.util.Set) HiveParserUtils.generateErrorMessage(org.apache.flink.table.planner.delegation.hive.HiveParserUtils.generateErrorMessage) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation) HiveASTParseUtils(org.apache.flink.table.planner.delegation.hive.copy.HiveASTParseUtils) HiveParserBaseSemanticAnalyzer.getGroupingSetsForCube(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getGroupingSetsForCube) SqlStdOperatorTable(org.apache.calcite.sql.fun.SqlStdOperatorTable) HiveParserPreCboCtx(org.apache.flink.table.planner.delegation.hive.copy.HiveParserPreCboCtx) RelCollation(org.apache.calcite.rel.RelCollation) HiveParserBaseSemanticAnalyzer.getGroupingSets(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getGroupingSets) HiveParserBaseSemanticAnalyzer.getPartitionKeys(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getPartitionKeys) HiveParserBaseSemanticAnalyzer.removeOBInSubQuery(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.removeOBInSubQuery) HiveASTParser(org.apache.flink.table.planner.delegation.hive.parse.HiveASTParser) ErrorMsg(org.apache.hadoop.hive.ql.ErrorMsg) RexCall(org.apache.calcite.rex.RexCall) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) TypeInfoUtils(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils) HiveShim(org.apache.flink.table.catalog.hive.client.HiveShim) HiveParserJoinTypeCheckCtx(org.apache.flink.table.planner.delegation.hive.copy.HiveParserJoinTypeCheckCtx) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) AggInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.AggInfo) HiveParserBaseSemanticAnalyzer.getCorrelationUse(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getCorrelationUse) HiveASTParseDriver(org.apache.flink.table.planner.delegation.hive.copy.HiveASTParseDriver) RexFieldCollation(org.apache.calcite.rex.RexFieldCollation) ViewExpanders(org.apache.calcite.plan.ViewExpanders) LogicalValues(org.apache.calcite.rel.logical.LogicalValues) LogicalCorrelate(org.apache.calcite.rel.logical.LogicalCorrelate) HiveParserBaseSemanticAnalyzer.validateNoHavingReferenceToAlias(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.validateNoHavingReferenceToAlias) HiveParserNamedJoinInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserNamedJoinInfo) RelDataType(org.apache.calcite.rel.type.RelDataType) HiveParserUtils.rewriteGroupingFunctionAST(org.apache.flink.table.planner.delegation.hive.HiveParserUtils.rewriteGroupingFunctionAST) LogicalIntersect(org.apache.calcite.rel.logical.LogicalIntersect) HiveParserQBSubQuery(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQBSubQuery) Table(org.apache.hadoop.hive.ql.metadata.Table) HiveParserRowResolver(org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver) Aggregate(org.apache.calcite.rel.core.Aggregate) HiveParserTypeCheckCtx(org.apache.flink.table.planner.delegation.hive.copy.HiveParserTypeCheckCtx) FrameworkConfig(org.apache.calcite.tools.FrameworkConfig) Node(org.apache.hadoop.hive.ql.lib.Node) HiveParserBaseSemanticAnalyzer.getBound(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getBound) HiveParserBaseSemanticAnalyzer.getColumnInternalName(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getColumnInternalName) HiveParserSqlFunctionConverter(org.apache.flink.table.planner.delegation.hive.copy.HiveParserSqlFunctionConverter) LogicalAggregate(org.apache.calcite.rel.logical.LogicalAggregate) JoinRelType(org.apache.calcite.rel.core.JoinRelType) AggregateCall(org.apache.calcite.rel.core.AggregateCall) SqlAggFunction(org.apache.calcite.sql.SqlAggFunction) ArrayDeque(java.util.ArrayDeque) RelDataTypeFactory(org.apache.calcite.rel.type.RelDataTypeFactory) ColumnAccessInfo(org.apache.hadoop.hive.ql.parse.ColumnAccessInfo) HiveParserBaseSemanticAnalyzer.obtainTableType(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.obtainTableType) HiveParserBaseSemanticAnalyzer.convert(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.convert) LogicalFilter(org.apache.calcite.rel.logical.LogicalFilter) RelFactories(org.apache.calcite.rel.core.RelFactories) LoggerFactory(org.slf4j.LoggerFactory) LogicalTableFunctionScan(org.apache.calcite.rel.logical.LogicalTableFunctionScan) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException) HiveParserBaseSemanticAnalyzer.processPositionAlias(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.processPositionAlias) HiveParserWindowingSpec(org.apache.flink.table.planner.delegation.hive.copy.HiveParserWindowingSpec) RexUtil(org.apache.calcite.rex.RexUtil) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) HiveParserErrorMsg(org.apache.flink.table.planner.delegation.hive.parse.HiveParserErrorMsg) RexNode(org.apache.calcite.rex.RexNode) LogicalUnion(org.apache.calcite.rel.logical.LogicalUnion) RelOptCluster(org.apache.calcite.plan.RelOptCluster) LogicalDistribution(org.apache.flink.table.planner.plan.nodes.hive.LogicalDistribution) RexLiteral(org.apache.calcite.rex.RexLiteral) HiveParserQBParseInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQBParseInfo) HiveParserTypeConverter(org.apache.flink.table.planner.delegation.hive.copy.HiveParserTypeConverter) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) Preconditions(org.apache.flink.util.Preconditions) CompositeList(org.apache.calcite.util.CompositeList) Collectors(java.util.stream.Collectors) RexInputRef(org.apache.calcite.rex.RexInputRef) VirtualColumn(org.apache.hadoop.hive.ql.metadata.VirtualColumn) List(java.util.List) Type(java.lang.reflect.Type) Sort(org.apache.calcite.rel.core.Sort) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) GenericUDAFEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator) HiveParserQueryState(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQueryState) Project(org.apache.calcite.rel.core.Project) CatalogManager(org.apache.flink.table.catalog.CatalogManager) HiveParserBaseSemanticAnalyzer.getGroupingSetsForRollup(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getGroupingSetsForRollup) HiveParserBaseSemanticAnalyzer.getWindowSpecIndx(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getWindowSpecIndx) HashMap(java.util.HashMap) SetOp(org.apache.calcite.rel.core.SetOp) Deque(java.util.Deque) RelOptUtil(org.apache.calcite.plan.RelOptUtil) LogicalMinus(org.apache.calcite.rel.logical.LogicalMinus) DeduplicateCorrelateVariables(org.apache.calcite.sql2rel.DeduplicateCorrelateVariables) HiveParserBaseSemanticAnalyzer.initPhase1Ctx(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.initPhase1Ctx) PlannerContext(org.apache.flink.table.planner.delegation.PlannerContext) HashSet(java.util.HashSet) HiveParserQBExpr(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQBExpr) TableType(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.TableType) Pair(org.apache.calcite.util.Pair) HiveParserBaseSemanticAnalyzer.buildHiveToCalciteColumnMap(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.buildHiveToCalciteColumnMap) SqlOperator(org.apache.calcite.sql.SqlOperator) FlinkCalciteCatalogReader(org.apache.flink.table.planner.plan.FlinkCalciteCatalogReader) RelCollations(org.apache.calcite.rel.RelCollations) ColumnInfo(org.apache.hadoop.hive.ql.exec.ColumnInfo) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) Logger(org.slf4j.Logger) HiveParserSemanticAnalyzer(org.apache.flink.table.planner.delegation.hive.copy.HiveParserSemanticAnalyzer) LogicalProject(org.apache.calcite.rel.logical.LogicalProject) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) TypeInfoFactory(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory) Iterator(java.util.Iterator) RexBuilder(org.apache.calcite.rex.RexBuilder) ExprNodeConstantDesc(org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc) GroupByDesc(org.apache.hadoop.hive.ql.plan.GroupByDesc) HiveConf(org.apache.hadoop.hive.conf.HiveConf) HiveParserQB(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQB) RelNode(org.apache.calcite.rel.RelNode) HiveParserBaseSemanticAnalyzer.genValues(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.genValues) HiveParserUtils.verifyCanHandleAst(org.apache.flink.table.planner.delegation.hive.HiveParserUtils.verifyCanHandleAst) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) AbstractMap(java.util.AbstractMap) ObjectPair(org.apache.hadoop.hive.common.ObjectPair) HiveParserBaseSemanticAnalyzer.buildHiveColNameToInputPosMap(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.buildHiveColNameToInputPosMap) HiveParserCreateViewInfo(org.apache.flink.table.planner.delegation.hive.parse.HiveParserCreateViewInfo) SqlUserDefinedTableFunction(org.apache.calcite.sql.validate.SqlUserDefinedTableFunction) HiveParserBaseSemanticAnalyzer.topLevelConjunctCheck(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.topLevelConjunctCheck) Util(org.apache.calcite.util.Util) HiveParserBaseSemanticAnalyzer.addToGBExpr(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.addToGBExpr) Collections(java.util.Collections) HiveParserBaseSemanticAnalyzer.getOrderKeys(org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer.getOrderKeys) HiveParserASTNode(org.apache.flink.table.planner.delegation.hive.copy.HiveParserASTNode) Node(org.apache.hadoop.hive.ql.lib.Node) RexNode(org.apache.calcite.rex.RexNode) RelNode(org.apache.calcite.rel.RelNode) ArrayList(java.util.ArrayList) ColumnInfo(org.apache.hadoop.hive.ql.exec.ColumnInfo) RelTraitSet(org.apache.calcite.plan.RelTraitSet) HiveParserRowResolver(org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver) LogicalSort(org.apache.calcite.rel.logical.LogicalSort) Sort(org.apache.calcite.rel.core.Sort) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException) Pair(org.apache.calcite.util.Pair) ObjectPair(org.apache.hadoop.hive.common.ObjectPair) HiveParserASTNode(org.apache.flink.table.planner.delegation.hive.copy.HiveParserASTNode) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) RelCollation(org.apache.calcite.rel.RelCollation) HiveParserQBParseInfo(org.apache.flink.table.planner.delegation.hive.copy.HiveParserQBParseInfo) RelNode(org.apache.calcite.rel.RelNode) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation) RexInputRef(org.apache.calcite.rex.RexInputRef) RexNode(org.apache.calcite.rex.RexNode)

Example 19 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project flink by apache.

the class FlinkRelMdCollation method project.

/**
 * Helper method to determine a {@link Project}'s collation.
 */
public static List<RelCollation> project(RelMetadataQuery mq, RelNode input, List<? extends RexNode> projects) {
    final SortedSet<RelCollation> collations = new TreeSet<>();
    final List<RelCollation> inputCollations = mq.collations(input);
    if (inputCollations == null || inputCollations.isEmpty()) {
        return com.google.common.collect.ImmutableList.of();
    }
    final com.google.common.collect.Multimap<Integer, Integer> targets = com.google.common.collect.LinkedListMultimap.create();
    final Map<Integer, SqlMonotonicity> targetsWithMonotonicity = new HashMap<>();
    for (Ord<RexNode> project : Ord.<RexNode>zip(projects)) {
        if (project.e instanceof RexInputRef) {
            targets.put(((RexInputRef) project.e).getIndex(), project.i);
        } else if (project.e instanceof RexCall) {
            final RexCall call = (RexCall) project.e;
            final RexCallBinding binding = RexCallBinding.create(input.getCluster().getTypeFactory(), call, inputCollations);
            targetsWithMonotonicity.put(project.i, call.getOperator().getMonotonicity(binding));
        }
    }
    final List<RelFieldCollation> fieldCollations = new ArrayList<>();
    loop: for (RelCollation ic : inputCollations) {
        if (ic.getFieldCollations().isEmpty()) {
            continue;
        }
        fieldCollations.clear();
        for (RelFieldCollation ifc : ic.getFieldCollations()) {
            final Collection<Integer> integers = targets.get(ifc.getFieldIndex());
            if (integers.isEmpty()) {
                // cannot do this collation
                continue loop;
            }
            fieldCollations.add(ifc.withFieldIndex(integers.iterator().next()));
        }
        assert !fieldCollations.isEmpty();
        collations.add(RelCollations.of(fieldCollations));
    }
    final List<RelFieldCollation> fieldCollationsForRexCalls = new ArrayList<>();
    for (Map.Entry<Integer, SqlMonotonicity> entry : targetsWithMonotonicity.entrySet()) {
        final SqlMonotonicity value = entry.getValue();
        switch(value) {
            case NOT_MONOTONIC:
            case CONSTANT:
                break;
            default:
                fieldCollationsForRexCalls.add(new RelFieldCollation(entry.getKey(), RelFieldCollation.Direction.of(value)));
                break;
        }
    }
    if (!fieldCollationsForRexCalls.isEmpty()) {
        collations.add(RelCollations.of(fieldCollationsForRexCalls));
    }
    return com.google.common.collect.ImmutableList.copyOf(collations);
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) RexCall(org.apache.calcite.rex.RexCall) RexCallBinding(org.apache.calcite.rex.RexCallBinding) RelCollation(org.apache.calcite.rel.RelCollation) TreeSet(java.util.TreeSet) SqlMonotonicity(org.apache.calcite.sql.validate.SqlMonotonicity) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation) RexInputRef(org.apache.calcite.rex.RexInputRef) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) RexNode(org.apache.calcite.rex.RexNode)

Example 20 with RelCollation

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollation in project flink by apache.

the class FlinkRelMdCollation method values.

/**
 * Helper method to determine a {@link org.apache.calcite.rel.core.Values}'s collation.
 *
 * <p>We actually under-report the collations. A Values with 0 or 1 rows - an edge case, but
 * legitimate and very common - is ordered by every permutation of every subset of the columns.
 *
 * <p>So, our algorithm aims to:
 *
 * <ul>
 *   <li>produce at most N collations (where N is the number of columns);
 *   <li>make each collation as long as possible;
 *   <li>do not repeat combinations already emitted - if we've emitted {@code (a, b)} do not
 *       later emit {@code (b, a)};
 *   <li>probe the actual values and make sure that each collation is consistent with the data
 * </ul>
 *
 * <p>So, for an empty Values with 4 columns, we would emit {@code (a, b, c, d), (b, c, d), (c,
 * d), (d)}.
 */
public static List<RelCollation> values(RelMetadataQuery mq, RelDataType rowType, com.google.common.collect.ImmutableList<com.google.common.collect.ImmutableList<RexLiteral>> tuples) {
    // for future use
    Util.discard(mq);
    final List<RelCollation> list = new ArrayList<>();
    final int n = rowType.getFieldCount();
    final List<Pair<RelFieldCollation, com.google.common.collect.Ordering<List<RexLiteral>>>> pairs = new ArrayList<>();
    outer: for (int i = 0; i < n; i++) {
        pairs.clear();
        for (int j = i; j < n; j++) {
            final RelFieldCollation fieldCollation = new RelFieldCollation(j);
            com.google.common.collect.Ordering<List<RexLiteral>> comparator = comparator(fieldCollation);
            com.google.common.collect.Ordering<List<RexLiteral>> ordering;
            if (pairs.isEmpty()) {
                ordering = comparator;
            } else {
                ordering = Util.last(pairs).right.compound(comparator);
            }
            pairs.add(Pair.of(fieldCollation, ordering));
            if (!ordering.isOrdered(tuples)) {
                if (j == i) {
                    continue outer;
                }
                pairs.remove(pairs.size() - 1);
            }
        }
        if (!pairs.isEmpty()) {
            list.add(RelCollations.of(Pair.left(pairs)));
        }
    }
    return list;
}
Also used : RexLiteral(org.apache.calcite.rex.RexLiteral) ArrayList(java.util.ArrayList) RelCollation(org.apache.calcite.rel.RelCollation) RelFieldCollation(org.apache.calcite.rel.RelFieldCollation) ImmutableIntList(org.apache.calcite.util.ImmutableIntList) List(java.util.List) ArrayList(java.util.ArrayList) Pair(org.apache.calcite.util.Pair)

Aggregations

RelCollation (org.apache.calcite.rel.RelCollation)83 RelNode (org.apache.calcite.rel.RelNode)40 RelTraitSet (org.apache.calcite.plan.RelTraitSet)37 RelFieldCollation (org.apache.calcite.rel.RelFieldCollation)33 RexNode (org.apache.calcite.rex.RexNode)24 ArrayList (java.util.ArrayList)19 RelDataType (org.apache.calcite.rel.type.RelDataType)17 RelOptCluster (org.apache.calcite.plan.RelOptCluster)15 ImmutableBitSet (org.apache.calcite.util.ImmutableBitSet)13 RelMetadataQuery (org.apache.calcite.rel.metadata.RelMetadataQuery)12 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)12 List (java.util.List)11 Sort (org.apache.calcite.rel.core.Sort)11 ImmutableList (com.google.common.collect.ImmutableList)8 RelDistribution (org.apache.calcite.rel.RelDistribution)8 RexInputRef (org.apache.calcite.rex.RexInputRef)8 Map (java.util.Map)7 Mappings (org.apache.calcite.util.mapping.Mappings)7 Supplier (com.google.common.base.Supplier)6 HashMap (java.util.HashMap)6