Examples with OrderExpression org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression used on opensource projects

Example 1 with OrderExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression in project hive by apache.

the class PTFTranslator method buildOrderExpressions.

/**
   * Collect order expressions for RANGE based windowing
   * @throws SemanticException
   */
private OrderDef buildOrderExpressions(ShapeDetails inpShape, List<OrderExpression> orderExpressions) throws SemanticException {
    OrderDef orderDef = new OrderDef();
    for (OrderExpression oe : orderExpressions) {
        PTFTranslator.validateNoLeadLagInValueBoundarySpec(oe.getExpression());
        PTFExpressionDef exprDef = null;
        try {
            exprDef = buildExpressionDef(inpShape, oe.getExpression());
        } catch (HiveException he) {
            throw new SemanticException(he);
        }
        PTFTranslator.validateValueBoundaryExprType(exprDef.getOI());
        OrderExpressionDef orderExprDef = new OrderExpressionDef(exprDef);
        orderExprDef.setOrder(oe.getOrder());
        orderExprDef.setNullOrder(oe.getNullOrder());
        orderDef.addExpression(orderExprDef);
    }
    return orderDef;
}

Example 2 with OrderExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression in project hive by apache.

the class SemanticAnalyzer method genReduceSinkPlanForWindowing.

private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, RowResolver inputRR, Operator input) throws SemanticException {
    ArrayList<ExprNodeDesc> partCols = new ArrayList<ExprNodeDesc>();
    ArrayList<ExprNodeDesc> orderCols = new ArrayList<ExprNodeDesc>();
    StringBuilder order = new StringBuilder();
    StringBuilder nullOrder = new StringBuilder();
    for (PartitionExpression partCol : spec.getQueryPartitionSpec().getExpressions()) {
        ExprNodeDesc partExpr = genExprNodeDesc(partCol.getExpression(), inputRR);
        if (ExprNodeDescUtils.indexOf(partExpr, partCols) < 0) {
            partCols.add(partExpr);
            orderCols.add(partExpr);
            order.append('+');
            nullOrder.append('a');
        }
    }
    if (spec.getQueryOrderSpec() != null) {
        for (OrderExpression orderCol : spec.getQueryOrderSpec().getExpressions()) {
            ExprNodeDesc orderExpr = genExprNodeDesc(orderCol.getExpression(), inputRR);
            char orderChar = orderCol.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-';
            char nullOrderChar = orderCol.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z';
            int index = ExprNodeDescUtils.indexOf(orderExpr, orderCols);
            if (index >= 0) {
                order.setCharAt(index, orderChar);
                nullOrder.setCharAt(index, nullOrderChar);
                continue;
            }
            orderCols.add(genExprNodeDesc(orderCol.getExpression(), inputRR));
            order.append(orderChar);
            nullOrder.append(nullOrderChar);
        }
    }
    return genReduceSinkPlan(input, partCols, orderCols, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID);
}

Example 3 with OrderExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression in project hive by apache.

the class SemanticAnalyzer method processOrderSpec.

private OrderSpec processOrderSpec(ASTNode sortNode) {
    OrderSpec oSpec = new OrderSpec();
    int exprCnt = sortNode.getChildCount();
    for (int i = 0; i < exprCnt; i++) {
        OrderExpression exprSpec = new OrderExpression();
        ASTNode orderSpec = (ASTNode) sortNode.getChild(i);
        ASTNode nullOrderSpec = (ASTNode) orderSpec.getChild(0);
        exprSpec.setExpression((ASTNode) nullOrderSpec.getChild(0));
        if (orderSpec.getType() == HiveParser.TOK_TABSORTCOLNAMEASC) {
            exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.ASC);
        } else {
            exprSpec.setOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order.DESC);
        }
        if (nullOrderSpec.getType() == HiveParser.TOK_NULLS_FIRST) {
            exprSpec.setNullOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder.NULLS_FIRST);
        } else {
            exprSpec.setNullOrder(org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder.NULLS_LAST);
        }
        oSpec.addExpression(exprSpec);
    }
    return oSpec;
}

Example 4 with OrderExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression in project hive by apache.

the class PTFTranslator method addPartitionExpressionsToOrderList.

protected static ArrayList<OrderExpression> addPartitionExpressionsToOrderList(ArrayList<PartitionExpression> partCols, ArrayList<OrderExpression> orderCols) throws SemanticException {
    int numOfPartColumns = 0;
    int chkSize = partCols.size();
    chkSize = chkSize > orderCols.size() ? orderCols.size() : chkSize;
    for (int i = 0; i < chkSize; i++) {
        if (orderCols.get(i).getExpression().toStringTree().equals(partCols.get(i).getExpression().toStringTree())) {
            numOfPartColumns++;
        } else {
            break;
        }
    }
    if (numOfPartColumns != 0 && numOfPartColumns != partCols.size()) {
        List<String> partitionColumnNames = new ArrayList<String>();
        for (PartitionExpression partitionExpression : partCols) {
            ASTNode column = partitionExpression.getExpression();
            if (column != null && column.getChildCount() > 0) {
                partitionColumnNames.add(column.getChild(0).getText());
            }
        }
        throw new SemanticException(String.format("all partition columns %s must be in order clause or none should be specified", partitionColumnNames.toString()));
    }
    ArrayList<OrderExpression> combinedOrdExprs = new ArrayList<OrderExpression>();
    if (numOfPartColumns == 0) {
        for (PartitionExpression partCol : partCols) {
            OrderExpression orderCol = new OrderExpression(partCol);
            combinedOrdExprs.add(orderCol);
        }
    }
    combinedOrdExprs.addAll(orderCols);
    return combinedOrdExprs;
}

Example 5 with OrderExpression

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression in project hive by apache.

the class HiveOpConverter method genPTF.

private OpAttr genPTF(OpAttr inputOpAf, WindowingSpec wSpec) throws SemanticException {
    Operator<?> input = inputOpAf.inputs.get(0);
    wSpec.validateAndMakeEffective();
    WindowingComponentizer groups = new WindowingComponentizer(wSpec);
    RowResolver rr = new RowResolver();
    for (ColumnInfo ci : input.getSchema().getSignature()) {
        rr.put(inputOpAf.tabAlias, ci.getInternalName(), ci);
    }
    while (groups.hasNext()) {
        wSpec = groups.next(hiveConf, semanticAnalyzer, unparseTranslator, rr);
        // 1. Create RS and backtrack Select operator on top
        ArrayList<ExprNodeDesc> keyCols = new ArrayList<ExprNodeDesc>();
        ArrayList<ExprNodeDesc> partCols = new ArrayList<ExprNodeDesc>();
        StringBuilder order = new StringBuilder();
        StringBuilder nullOrder = new StringBuilder();
        for (PartitionExpression partCol : wSpec.getQueryPartitionSpec().getExpressions()) {
            ExprNodeDesc partExpr = semanticAnalyzer.genExprNodeDesc(partCol.getExpression(), rr);
            if (ExprNodeDescUtils.indexOf(partExpr, partCols) < 0) {
                keyCols.add(partExpr);
                partCols.add(partExpr);
                order.append('+');
                nullOrder.append('a');
            }
        }
        if (wSpec.getQueryOrderSpec() != null) {
            for (OrderExpression orderCol : wSpec.getQueryOrderSpec().getExpressions()) {
                ExprNodeDesc orderExpr = semanticAnalyzer.genExprNodeDesc(orderCol.getExpression(), rr);
                char orderChar = orderCol.getOrder() == PTFInvocationSpec.Order.ASC ? '+' : '-';
                char nullOrderChar = orderCol.getNullOrder() == PTFInvocationSpec.NullOrder.NULLS_FIRST ? 'a' : 'z';
                int index = ExprNodeDescUtils.indexOf(orderExpr, keyCols);
                if (index >= 0) {
                    order.setCharAt(index, orderChar);
                    nullOrder.setCharAt(index, nullOrderChar);
                    continue;
                }
                keyCols.add(orderExpr);
                order.append(orderChar);
                nullOrder.append(nullOrderChar);
            }
        }
        SelectOperator selectOp = genReduceSinkAndBacktrackSelect(input, keyCols.toArray(new ExprNodeDesc[keyCols.size()]), 0, partCols, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID, hiveConf);
        // 2. Finally create PTF
        PTFTranslator translator = new PTFTranslator();
        PTFDesc ptfDesc = translator.translate(wSpec, semanticAnalyzer, hiveConf, rr, unparseTranslator);
        RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr();
        Operator<?> ptfOp = OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()), selectOp);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Generated " + ptfOp + " with row schema: [" + ptfOp.getSchema() + "]");
        }
        // 3. Prepare for next iteration (if any)
        rr = ptfOpRR;
        input = ptfOp;
    }
    return inputOpAf.clone(input);
}