Search in sources :

Example 1 with PartitioningSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec in project hive by apache.

the class SemanticAnalyzer method processPTFPartitionSpec.

private PartitioningSpec processPTFPartitionSpec(ASTNode pSpecNode) {
    PartitioningSpec partitioning = new PartitioningSpec();
    ASTNode firstChild = (ASTNode) pSpecNode.getChild(0);
    int type = firstChild.getType();
    if (type == HiveParser.TOK_DISTRIBUTEBY || type == HiveParser.TOK_CLUSTERBY) {
        PartitionSpec pSpec = processPartitionSpec(firstChild);
        partitioning.setPartSpec(pSpec);
        ASTNode sortNode = pSpecNode.getChildCount() > 1 ? (ASTNode) pSpecNode.getChild(1) : null;
        if (sortNode != null) {
            OrderSpec oSpec = processOrderSpec(sortNode);
            partitioning.setOrderSpec(oSpec);
        }
    } else if (type == HiveParser.TOK_SORTBY || type == HiveParser.TOK_ORDERBY) {
        OrderSpec oSpec = processOrderSpec(firstChild);
        partitioning.setOrderSpec(oSpec);
    }
    return partitioning;
}
Also used : OrderSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec) PartitionSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec) SQLUniqueConstraint(org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint) SQLCheckConstraint(org.apache.hadoop.hive.metastore.api.SQLCheckConstraint) SQLDefaultConstraint(org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint) DefaultConstraint(org.apache.hadoop.hive.ql.metadata.DefaultConstraint) SQLNotNullConstraint(org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint) PartitioningSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec)

Example 2 with PartitioningSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec in project hive by apache.

the class ExprNodeConverter method visitOver.

@Override
public ExprNodeDesc visitOver(RexOver over) {
    if (!deep) {
        return null;
    }
    final RexWindow window = over.getWindow();
    final WindowSpec windowSpec = new WindowSpec();
    final PartitioningSpec partitioningSpec = getPSpec(window);
    windowSpec.setPartitioning(partitioningSpec);
    final WindowFrameSpec windowFrameSpec = getWindowRange(window);
    windowSpec.setWindowFrame(windowFrameSpec);
    WindowFunctionSpec wfs = new WindowFunctionSpec();
    wfs.setWindowSpec(windowSpec);
    final Schema schema = new Schema(tabAlias, inputRowType.getFieldList());
    final ASTNode wUDAFAst = new ASTConverter.RexVisitor(schema).visitOver(over);
    wfs.setExpression(wUDAFAst);
    ASTNode nameNode = (ASTNode) wUDAFAst.getChild(0);
    wfs.setName(nameNode.getText());
    for (int i = 1; i < wUDAFAst.getChildCount() - 1; i++) {
        ASTNode child = (ASTNode) wUDAFAst.getChild(i);
        wfs.addArg(child);
    }
    if (wUDAFAst.getText().equals("TOK_FUNCTIONSTAR")) {
        wfs.setStar(true);
    }
    String columnAlias = getWindowColumnAlias();
    wfs.setAlias(columnAlias);
    this.windowFunctionSpecs.add(wfs);
    return new ExprNodeColumnDesc(TypeConverter.convert(over.getType()), columnAlias, tabAlias, false);
}
Also used : RexVisitor(org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor) Schema(org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema) ASTNode(org.apache.hadoop.hive.ql.parse.ASTNode) ExprNodeColumnDesc(org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc) WindowFunctionSpec(org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec) RexWindow(org.apache.calcite.rex.RexWindow) DateString(org.apache.calcite.util.DateString) ByteString(org.apache.calcite.avatica.util.ByteString) TimestampString(org.apache.calcite.util.TimestampString) TimeString(org.apache.calcite.util.TimeString) NlsString(org.apache.calcite.util.NlsString) WindowSpec(org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec) PartitioningSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec) WindowFrameSpec(org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFrameSpec)

Example 3 with PartitioningSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec in project hive by apache.

the class ExprNodeConverter method getPSpec.

private PartitioningSpec getPSpec(RexWindow window) {
    PartitioningSpec partitioning = new PartitioningSpec();
    Schema schema = new Schema(tabAlias, inputRowType.getFieldList());
    if (window.partitionKeys != null && !window.partitionKeys.isEmpty()) {
        PartitionSpec pSpec = new PartitionSpec();
        for (RexNode pk : window.partitionKeys) {
            PartitionExpression exprSpec = new PartitionExpression();
            ASTNode astNode = pk.accept(new RexVisitor(schema));
            exprSpec.setExpression(astNode);
            pSpec.addExpression(exprSpec);
        }
        partitioning.setPartSpec(pSpec);
    }
    if (window.orderKeys != null && !window.orderKeys.isEmpty()) {
        OrderSpec oSpec = new OrderSpec();
        for (RexFieldCollation ok : window.orderKeys) {
            OrderExpression exprSpec = new OrderExpression();
            Order order = ok.getDirection() == RelFieldCollation.Direction.ASCENDING ? Order.ASC : Order.DESC;
            NullOrder nullOrder;
            if (ok.right.contains(SqlKind.NULLS_FIRST)) {
                nullOrder = NullOrder.NULLS_FIRST;
            } else if (ok.right.contains(SqlKind.NULLS_LAST)) {
                nullOrder = NullOrder.NULLS_LAST;
            } else {
                // Default
                nullOrder = ok.getDirection() == RelFieldCollation.Direction.ASCENDING ? NullOrder.NULLS_FIRST : NullOrder.NULLS_LAST;
            }
            exprSpec.setOrder(order);
            exprSpec.setNullOrder(nullOrder);
            ASTNode astNode = ok.left.accept(new RexVisitor(schema));
            exprSpec.setExpression(astNode);
            oSpec.addExpression(exprSpec);
        }
        partitioning.setOrderSpec(oSpec);
    }
    return partitioning;
}
Also used : NullOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder) Order(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order) OrderSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec) RexVisitor(org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor) PartitionExpression(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression) OrderExpression(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression) Schema(org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema) ASTNode(org.apache.hadoop.hive.ql.parse.ASTNode) NullOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder) PartitionSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec) RexFieldCollation(org.apache.calcite.rex.RexFieldCollation) PartitioningSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec) RexNode(org.apache.calcite.rex.RexNode)

Example 4 with PartitioningSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec in project hive by apache.

the class SemanticAnalyzer method processPTFChain.

/*
   * - tree form is
   *   ^(TOK_PTBLFUNCTION name alias? partitionTableFunctionSource partitioningSpec? arguments*)
   * - a partitionTableFunctionSource can be a tableReference, a SubQuery or another
   *   PTF invocation.
   */
private PartitionedTableFunctionSpec processPTFChain(QB qb, ASTNode ptf) throws SemanticException {
    int child_count = ptf.getChildCount();
    if (child_count < 2) {
        throw new SemanticException(generateErrorMessage(ptf, "Not enough Children " + child_count));
    }
    PartitionedTableFunctionSpec ptfSpec = new PartitionedTableFunctionSpec();
    ptfSpec.setAstNode(ptf);
    /*
     * name
     */
    ASTNode nameNode = (ASTNode) ptf.getChild(0);
    ptfSpec.setName(nameNode.getText());
    int inputIdx = 1;
    /*
     * alias
     */
    ASTNode secondChild = (ASTNode) ptf.getChild(1);
    if (secondChild.getType() == HiveParser.Identifier) {
        ptfSpec.setAlias(secondChild.getText());
        inputIdx++;
    }
    /*
     * input
     */
    ASTNode inputNode = (ASTNode) ptf.getChild(inputIdx);
    ptfSpec.setInput(processPTFSource(qb, inputNode));
    int argStartIdx = inputIdx + 1;
    /*
     * partitioning Spec
     */
    int pSpecIdx = inputIdx + 1;
    ASTNode pSpecNode = ptf.getChildCount() > inputIdx ? (ASTNode) ptf.getChild(pSpecIdx) : null;
    if (pSpecNode != null && pSpecNode.getType() == HiveParser.TOK_PARTITIONINGSPEC) {
        PartitioningSpec partitioning = processPTFPartitionSpec(pSpecNode);
        ptfSpec.setPartitioning(partitioning);
        argStartIdx++;
    }
    /*
     * arguments
     */
    for (int i = argStartIdx; i < ptf.getChildCount(); i++) {
        ptfSpec.addArg((ASTNode) ptf.getChild(i));
    }
    return ptfSpec;
}
Also used : PartitionedTableFunctionSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionedTableFunctionSpec) SQLUniqueConstraint(org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint) SQLCheckConstraint(org.apache.hadoop.hive.metastore.api.SQLCheckConstraint) SQLDefaultConstraint(org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint) DefaultConstraint(org.apache.hadoop.hive.ql.metadata.DefaultConstraint) SQLNotNullConstraint(org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint) CalciteSemanticException(org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException) PartitioningSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec)

Example 5 with PartitioningSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec in project hive by apache.

the class WindowingComponentizer method groupFunctions.

private void groupFunctions() throws SemanticException {
    for (WindowExpressionSpec expr : originalSpec.getWindowExpressions()) {
        WindowFunctionSpec wFn = (WindowFunctionSpec) expr;
        PartitioningSpec wFnGrp = wFn.getWindowSpec().getPartitioning();
        WindowingSpec wSpec = groups.get(wFnGrp);
        if (wSpec == null) {
            wSpec = new WindowingSpec();
            groups.put(wFnGrp, wSpec);
        }
        wSpec.addWindowFunction(wFn);
    }
}
Also used : WindowFunctionSpec(org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec) WindowExpressionSpec(org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec) PartitioningSpec(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec)

Aggregations

PartitioningSpec (org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec)7 SQLCheckConstraint (org.apache.hadoop.hive.metastore.api.SQLCheckConstraint)3 SQLDefaultConstraint (org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint)3 SQLNotNullConstraint (org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint)3 SQLUniqueConstraint (org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint)3 DefaultConstraint (org.apache.hadoop.hive.ql.metadata.DefaultConstraint)3 WindowFunctionSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec)3 RexVisitor (org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor)2 Schema (org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema)2 ASTNode (org.apache.hadoop.hive.ql.parse.ASTNode)2 OrderSpec (org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec)2 PartitionSpec (org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec)2 WindowExpressionSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec)2 WindowFrameSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFrameSpec)2 WindowSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec)2 ArrayList (java.util.ArrayList)1 ByteString (org.apache.calcite.avatica.util.ByteString)1 RexFieldCollation (org.apache.calcite.rex.RexFieldCollation)1 RexNode (org.apache.calcite.rex.RexNode)1 RexWindow (org.apache.calcite.rex.RexWindow)1