Examples with WindowFunctionSpec org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec used on opensource projects

Example 1 with WindowFunctionSpec

use of org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec in project hive by apache.

the class SemanticAnalyzer method processWindowFunction.

private WindowFunctionSpec processWindowFunction(ASTNode node, ASTNode wsNode) throws SemanticException {
    WindowFunctionSpec wfSpec = new WindowFunctionSpec();
    switch(node.getType()) {
        case HiveParser.TOK_FUNCTIONSTAR:
            wfSpec.setStar(true);
            break;
        case HiveParser.TOK_FUNCTIONDI:
            wfSpec.setDistinct(true);
            break;
    }
    wfSpec.setExpression(node);
    ASTNode nameNode = (ASTNode) node.getChild(0);
    wfSpec.setName(nameNode.getText());
    for (int i = 1; i < node.getChildCount() - 1; i++) {
        ASTNode child = (ASTNode) node.getChild(i);
        wfSpec.addArg(child);
    }
    if (wsNode != null) {
        WindowSpec ws = processWindowSpec(wsNode);
        wfSpec.setWindowSpec(ws);
    }
    return wfSpec;
}

Example 2 with WindowFunctionSpec

use of org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec in project hive by apache.

the class HiveOpConverter method visit.

OpAttr visit(HiveProject projectRel) throws SemanticException {
    OpAttr inputOpAf = dispatch(projectRel.getInput());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + projectRel.getId() + ":" + projectRel.getRelTypeName() + " with row type: [" + projectRel.getRowType() + "]");
    }
    WindowingSpec windowingSpec = new WindowingSpec();
    List<String> exprNames = new ArrayList<String>(projectRel.getRowType().getFieldNames());
    List<ExprNodeDesc> exprCols = new ArrayList<ExprNodeDesc>();
    Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
    for (int pos = 0; pos < projectRel.getChildExps().size(); pos++) {
        ExprNodeConverter converter = new ExprNodeConverter(inputOpAf.tabAlias, projectRel.getRowType().getFieldNames().get(pos), projectRel.getInput().getRowType(), projectRel.getRowType(), inputOpAf.vcolsInCalcite, projectRel.getCluster().getTypeFactory(), true);
        ExprNodeDesc exprCol = projectRel.getChildExps().get(pos).accept(converter);
        colExprMap.put(exprNames.get(pos), exprCol);
        exprCols.add(exprCol);
        //TODO: Cols that come through PTF should it retain (VirtualColumness)?
        if (converter.getWindowFunctionSpec() != null) {
            for (WindowFunctionSpec wfs : converter.getWindowFunctionSpec()) {
                windowingSpec.addWindowFunction(wfs);
            }
        }
    }
    if (windowingSpec.getWindowExpressions() != null && !windowingSpec.getWindowExpressions().isEmpty()) {
        inputOpAf = genPTF(inputOpAf, windowingSpec);
    }
    // TODO: is this a safe assumption (name collision, external names...)
    SelectDesc sd = new SelectDesc(exprCols, exprNames);
    Pair<ArrayList<ColumnInfo>, Set<Integer>> colInfoVColPair = createColInfos(projectRel.getChildExps(), exprCols, exprNames, inputOpAf);
    SelectOperator selOp = (SelectOperator) OperatorFactory.getAndMakeChild(sd, new RowSchema(colInfoVColPair.getKey()), inputOpAf.inputs.get(0));
    selOp.setColumnExprMap(colExprMap);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated " + selOp + " with row schema: [" + selOp.getSchema() + "]");
    }
    return new OpAttr(inputOpAf.tabAlias, colInfoVColPair.getValue(), selOp);
}

Example 3 with WindowFunctionSpec

use of org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec in project hive by apache.

the class ExprNodeConverter method visitOver.

@Override
public ExprNodeDesc visitOver(RexOver over) {
    if (!deep) {
        return null;
    }
    final RexWindow window = over.getWindow();
    final WindowSpec windowSpec = new WindowSpec();
    final PartitioningSpec partitioningSpec = getPSpec(window);
    windowSpec.setPartitioning(partitioningSpec);
    final WindowFrameSpec windowFrameSpec = getWindowRange(window);
    windowSpec.setWindowFrame(windowFrameSpec);
    WindowFunctionSpec wfs = new WindowFunctionSpec();
    wfs.setWindowSpec(windowSpec);
    final Schema schema = new Schema(tabAlias, inputRowType.getFieldList());
    final ASTNode wUDAFAst = new ASTConverter.RexVisitor(schema).visitOver(over);
    wfs.setExpression(wUDAFAst);
    ASTNode nameNode = (ASTNode) wUDAFAst.getChild(0);
    wfs.setName(nameNode.getText());
    for (int i = 1; i < wUDAFAst.getChildCount() - 1; i++) {
        ASTNode child = (ASTNode) wUDAFAst.getChild(i);
        wfs.addArg(child);
    }
    if (wUDAFAst.getText().equals("TOK_FUNCTIONSTAR")) {
        wfs.setStar(true);
    }
    String columnAlias = getWindowColumnAlias();
    wfs.setAlias(columnAlias);
    this.windowFunctionSpecs.add(wfs);
    return new ExprNodeColumnDesc(TypeConverter.convert(over.getType()), columnAlias, tabAlias, false);
}

Example 4 with WindowFunctionSpec

use of org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec in project hive by apache.

the class PTFTranslator method translate.

public PTFDesc translate(WindowingSpec wdwSpec, SemanticAnalyzer semAly, HiveConf hCfg, RowResolver inputRR, UnparseTranslator unparseT) throws SemanticException {
    init(semAly, hCfg, inputRR, unparseT);
    windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);
    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();
    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver) FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
        throw new SemanticException(String.format("Internal Error: Unknown Table Function %s", FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);
    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if (partiSpec == null) {
        throw new SemanticException("Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);
    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);
    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if (wdwSpec.getWindowExpressions() != null) {
        for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
            if (expr instanceof WindowFunctionSpec) {
                WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec) expr);
                windowFunctions.add(wFnDef);
            }
        }
        wdwTFnDef.setWindowFunctions(windowFunctions);
    }
    /*
     * set outputFromWdwFnProcessing
     */
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : windowFunctions) {
        aliases.add(wFnDef.getAlias());
        if (wFnDef.isPivotResult()) {
            fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
        } else {
            fieldOIs.add(wFnDef.getOI());
        }
    }
    PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
    tFn.setWdwProcessingOutputOI(wdwOutOI);
    RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
    ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
    wdwTFnDef.setOutputShape(wdwOutShape);
    tFn.setupOutputOI();
    PTFDeserializer.alterOutputOIForStreaming(ptfDesc);
    return ptfDesc;
}

Example 5 with WindowFunctionSpec

use of org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec in project hive by apache.

the class SemanticAnalyzer method doPhase1GetAggregationsFromSelect.

private LinkedHashMap<String, ASTNode> doPhase1GetAggregationsFromSelect(ASTNode selExpr, QB qb, String dest) throws SemanticException {
    // Iterate over the selects search for aggregation Trees.
    // Use String as keys to eliminate duplicate trees.
    LinkedHashMap<String, ASTNode> aggregationTrees = new LinkedHashMap<String, ASTNode>();
    List<ASTNode> wdwFns = new ArrayList<ASTNode>();
    for (int i = 0; i < selExpr.getChildCount(); ++i) {
        ASTNode function = (ASTNode) selExpr.getChild(i);
        if (function.getType() == HiveParser.TOK_SELEXPR || function.getType() == HiveParser.TOK_SUBQUERY_EXPR) {
            function = (ASTNode) function.getChild(0);
        }
        doPhase1GetAllAggregations(function, aggregationTrees, wdwFns);
    }
    // window based aggregations are handled differently
    for (ASTNode wdwFn : wdwFns) {
        WindowingSpec spec = qb.getWindowingSpec(dest);
        if (spec == null) {
            queryProperties.setHasWindowing(true);
            spec = new WindowingSpec();
            qb.addDestToWindowingSpec(dest, spec);
        }
        HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
        int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
        WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn, (ASTNode) wdwFn.getChild(wdwFn.getChildCount() - 1));
        // If this is a duplicate invocation of a function; don't add to WindowingSpec.
        if (wExprsInDest != null && wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
            continue;
        }
        wFnSpec.setAlias(wFnSpec.getName() + "_window_" + wColIdx);
        spec.addWindowFunction(wFnSpec);
        qb.getParseInfo().addWindowingExprToClause(dest, wFnSpec.getExpression());
    }
    return aggregationTrees;
}