Examples with PartitionExpression org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression used on opensource projects

Example 6 with PartitionExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression in project hive by apache.

the class HiveOpConverter method genPTF.

private OpAttr genPTF(OpAttr inputOpAf, WindowingSpec wSpec) throws SemanticException {
    Operator<?> input = inputOpAf.inputs.get(0);
    wSpec.validateAndMakeEffective();
    WindowingComponentizer groups = new WindowingComponentizer(wSpec);
    RowResolver rr = new RowResolver();
    for (ColumnInfo ci : input.getSchema().getSignature()) {
        rr.put(inputOpAf.tabAlias, ci.getInternalName(), ci);
    }
    while (groups.hasNext()) {
        wSpec = groups.next(hiveConf, semanticAnalyzer, unparseTranslator, rr);
        // 1. Create RS and backtrack Select operator on top
        ArrayList<ExprNodeDesc> keyCols = new ArrayList<ExprNodeDesc>();
        ArrayList<ExprNodeDesc> partCols = new ArrayList<ExprNodeDesc>();
        StringBuilder order = new StringBuilder();
        StringBuilder nullOrder = new StringBuilder();
        for (PartitionExpression partCol : wSpec.getQueryPartitionSpec().getExpressions()) {
            ExprNodeDesc partExpr = semanticAnalyzer.genExprNodeDesc(partCol.getExpression(), rr);
            if (ExprNodeDescUtils.indexOf(partExpr, partCols) < 0) {
                keyCols.add(partExpr);
                partCols.add(partExpr);
                order.append('+');
                nullOrder.append('a');
            }
        }
        if (wSpec.getQueryOrderSpec() != null) {
            for (OrderExpression orderCol : wSpec.getQueryOrderSpec().getExpressions()) {
                ExprNodeDesc orderExpr = semanticAnalyzer.genExprNodeDesc(orderCol.getExpression(), rr);
                char orderChar = orderCol.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-';
                char nullOrderChar = orderCol.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z';
                int index = ExprNodeDescUtils.indexOf(orderExpr, keyCols);
                if (index >= 0) {
                    order.setCharAt(index, orderChar);
                    nullOrder.setCharAt(index, nullOrderChar);
                    continue;
                }
                keyCols.add(orderExpr);
                order.append(orderChar);
                nullOrder.append(nullOrderChar);
            }
        }
        SelectOperator selectOp = genReduceSinkAndBacktrackSelect(input, keyCols.toArray(new ExprNodeDesc[keyCols.size()]), 0, partCols, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID, hiveConf);
        // 2. Finally create PTF
        PTFTranslator translator = new PTFTranslator();
        PTFDesc ptfDesc = translator.translate(wSpec, semanticAnalyzer, hiveConf, rr, unparseTranslator);
        RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
        Operator<?> ptfOp = OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), selectOp);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Generated " + ptfOp + " with row schema: [" + ptfOp.getSchema() + "]");
        }
        // 3. Prepare for next iteration (if any)
        rr = ptfOpRR;
        input = ptfOp;
    }
    return inputOpAf.clone(input);
}

Example 7 with PartitionExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression in project hive by apache.

the class ExprNodeConverter method getPSpec.

private PartitioningSpec getPSpec(RexWindow window) {
    PartitioningSpec partitioning = new PartitioningSpec();
    Schema schema = new Schema(tabAlias, inputRowType.getFieldList());
    if (window.partitionKeys != null && !window.partitionKeys.isEmpty()) {
        PartitionSpec pSpec = new PartitionSpec();
        for (RexNode pk : window.partitionKeys) {
            PartitionExpression exprSpec = new PartitionExpression();
            ASTNode astNode = pk.accept(new RexVisitor(schema));
            exprSpec.setExpression(astNode);
            pSpec.addExpression(exprSpec);
        }
        partitioning.setPartSpec(pSpec);
    }
    if (window.orderKeys != null && !window.orderKeys.isEmpty()) {
        OrderSpec oSpec = new OrderSpec();
        for (RexFieldCollation ok : window.orderKeys) {
            OrderExpression exprSpec = new OrderExpression();
            Order order = ok.getDirection() == RelFieldCollation.Direction.ASCENDING ? Order.ASC : Order.DESC;
            NullOrder nullOrder;
            if (ok.right.contains(SqlKind.NULLS_FIRST)) {
                nullOrder = NullOrder.NULLS_FIRST;
            } else if (ok.right.contains(SqlKind.NULLS_LAST)) {
                nullOrder = NullOrder.NULLS_LAST;
            } else {
                // Default
                nullOrder = ok.getDirection() == RelFieldCollation.Direction.ASCENDING ? NullOrder.NULLS_FIRST : NullOrder.NULLS_LAST;
            }
            exprSpec.setOrder(order);
            exprSpec.setNullOrder(nullOrder);
            ASTNode astNode = ok.left.accept(new RexVisitor(schema));
            exprSpec.setExpression(astNode);
            oSpec.addExpression(exprSpec);
        }
        partitioning.setOrderSpec(oSpec);
    }
    return partitioning;
}

Example 8 with PartitionExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression in project hive by apache.

the class SemanticAnalyzer method processPartitionSpec.

private PartitionSpec processPartitionSpec(ASTNode node) {
    PartitionSpec pSpec = new PartitionSpec();
    int exprCnt = node.getChildCount();
    for (int i = 0; i < exprCnt; i++) {
        PartitionExpression exprSpec = new PartitionExpression();
        exprSpec.setExpression((ASTNode) node.getChild(i));
        pSpec.addExpression(exprSpec);
    }
    return pSpec;
}