Examples with PTFQueryInputSpec org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec used on opensource projects

Example 1 with PTFQueryInputSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec in project hive by apache.

the class SemanticAnalyzer method processPTFSource.

// --------------------------- PTF handling -----------------------------------
/*
   * - a partitionTableFunctionSource can be a tableReference, a SubQuery or another
   *   PTF invocation.
   * - For a TABLEREF: set the source to the alias returned by processTable
   * - For a SubQuery: set the source to the alias returned by processSubQuery
   * - For a PTF invocation: recursively call processPTFChain.
   */
private PTFInputSpec processPTFSource(QB qb, ASTNode inputNode) throws SemanticException {
    PTFInputSpec qInSpec = null;
    int type = inputNode.getType();
    String alias;
    switch(type) {
        case HiveParser.TOK_TABREF:
            alias = processTable(qb, inputNode);
            qInSpec = new PTFQueryInputSpec();
            ((PTFQueryInputSpec) qInSpec).setType(PTFQueryInputType.TABLE);
            ((PTFQueryInputSpec) qInSpec).setSource(alias);
            break;
        case HiveParser.TOK_SUBQUERY:
            alias = processSubQuery(qb, inputNode);
            qInSpec = new PTFQueryInputSpec();
            ((PTFQueryInputSpec) qInSpec).setType(PTFQueryInputType.SUBQUERY);
            ((PTFQueryInputSpec) qInSpec).setSource(alias);
            break;
        case HiveParser.TOK_PTBLFUNCTION:
            qInSpec = processPTFChain(qb, inputNode);
            break;
        default:
            throw new SemanticException(generateErrorMessage(inputNode, "Unknown input type to PTF"));
    }
    qInSpec.setAstNode(inputNode);
    return qInSpec;
}

Example 2 with PTFQueryInputSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec in project hive by apache.

the class PTFTranslator method translatePTFChain.

private void translatePTFChain() throws SemanticException {
    Deque<PTFInputSpec> ptfChain = new ArrayDeque<PTFInvocationSpec.PTFInputSpec>();
    PTFInputSpec currentSpec = ptfInvocation.getFunction();
    while (currentSpec != null) {
        ptfChain.push(currentSpec);
        currentSpec = currentSpec.getInput();
    }
    int inputNum = 0;
    PTFInputDef currentDef = null;
    while (!ptfChain.isEmpty()) {
        currentSpec = ptfChain.pop();
        if (currentSpec instanceof PTFQueryInputSpec) {
            currentDef = translate((PTFQueryInputSpec) currentSpec, inputNum);
        } else {
            currentDef = translate((PartitionedTableFunctionSpec) currentSpec, currentDef, inputNum);
        }
        inputNum++;
    }
    ptfDesc.setFuncDef((PartitionedTableFunctionDef) currentDef);
}

Example 3 with PTFQueryInputSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec in project hive by apache.

the class PTFTranslator method componentize.

public static ArrayList<PTFInvocationSpec> componentize(PTFInvocationSpec ptfInvocation) throws SemanticException {
    ArrayList<PTFInvocationSpec> componentInvocations = new ArrayList<PTFInvocationSpec>();
    Stack<PTFInputSpec> ptfChain = new Stack<PTFInvocationSpec.PTFInputSpec>();
    PTFInputSpec spec = ptfInvocation.getFunction();
    while (spec instanceof PartitionedTableFunctionSpec) {
        ptfChain.push(spec);
        spec = spec.getInput();
    }
    PartitionedTableFunctionSpec prevFn = (PartitionedTableFunctionSpec) ptfChain.pop();
    applyConstantPartition(prevFn);
    PartitionSpec partSpec = prevFn.getPartition();
    OrderSpec orderSpec = prevFn.getOrder();
    if (partSpec == null) {
        // oops this should have been caught before trying to componentize
        throw new SemanticException("No Partitioning specification specified at start of a PTFChain");
    }
    if (orderSpec == null) {
        orderSpec = new OrderSpec(partSpec);
        prevFn.setOrder(orderSpec);
    }
    while (!ptfChain.isEmpty()) {
        PartitionedTableFunctionSpec currentFn = (PartitionedTableFunctionSpec) ptfChain.pop();
        String fnName = currentFn.getName();
        if (!FunctionRegistry.isTableFunction(fnName)) {
            throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(fnName));
        }
        boolean transformsRawInput = FunctionRegistry.getTableFunctionResolver(fnName).transformsRawInput();
        /*
       * if the current table function has no partition info specified: inherit it from the PTF up
       * the chain.
       */
        if (currentFn.getPartition() == null) {
            currentFn.setPartition(prevFn.getPartition());
            if (currentFn.getOrder() == null) {
                currentFn.setOrder(prevFn.getOrder());
            }
        }
        /*
       * If the current table function has no order info specified;
       */
        if (currentFn.getOrder() == null) {
            currentFn.setOrder(new OrderSpec(currentFn.getPartition()));
        }
        if (!currentFn.getPartition().equals(partSpec) || !currentFn.getOrder().equals(orderSpec) || transformsRawInput) {
            PTFInvocationSpec component = new PTFInvocationSpec();
            component.setFunction(prevFn);
            componentInvocations.add(component);
            PTFQueryInputSpec cQInSpec = new PTFQueryInputSpec();
            cQInSpec.setType(PTFQueryInputType.PTFCOMPONENT);
            currentFn.setInput(cQInSpec);
        }
        prevFn = currentFn;
        partSpec = prevFn.getPartition();
        orderSpec = prevFn.getOrder();
    }
    componentInvocations.add(ptfInvocation);
    return componentInvocations;
}

Example 4 with PTFQueryInputSpec

use of org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec in project hive by apache.

the class PTFTranslator method translate.

public PTFDesc translate(WindowingSpec wdwSpec, SemanticAnalyzer semAly, HiveConf hCfg, RowResolver inputRR, UnparseTranslator unparseT) throws SemanticException {
    init(semAly, hCfg, inputRR, unparseT);
    windowingSpec = wdwSpec;
    ptfDesc = new PTFDesc();
    ptfDesc.setCfg(hCfg);
    ptfDesc.setLlInfo(llInfo);
    WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
    ptfDesc.setFuncDef(wdwTFnDef);
    PTFQueryInputSpec inpSpec = new PTFQueryInputSpec();
    inpSpec.setType(PTFQueryInputType.WINDOWING);
    wdwTFnDef.setInput(translate(inpSpec, 0));
    ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape();
    WindowingTableFunctionResolver tFn = (WindowingTableFunctionResolver) FunctionRegistry.getTableFunctionResolver(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    if (tFn == null) {
        throw new SemanticException(String.format("Internal Error: Unknown Table Function %s", FunctionRegistry.WINDOWING_TABLE_FUNCTION));
    }
    wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION);
    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);
    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();
    if (partiSpec == null) {
        throw new SemanticException("Invalid use of Windowing: there is no Partitioning associated with Windowing");
    }
    PartitionDef partDef = translate(inpShape, wdwSpec.getQueryPartitionSpec());
    OrderDef ordDef = translate(inpShape, wdwSpec.getQueryOrderSpec(), partDef);
    wdwTFnDef.setPartition(partDef);
    wdwTFnDef.setOrder(ordDef);
    /*
     * process Wdw functions
     */
    ArrayList<WindowFunctionDef> windowFunctions = new ArrayList<WindowFunctionDef>();
    if (wdwSpec.getWindowExpressions() != null) {
        for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
            if (expr instanceof WindowFunctionSpec) {
                WindowFunctionDef wFnDef = translate(wdwTFnDef, (WindowFunctionSpec) expr);
                windowFunctions.add(wFnDef);
            }
        }
        wdwTFnDef.setWindowFunctions(windowFunctions);
    }
    /*
     * set outputFromWdwFnProcessing
     */
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : windowFunctions) {
        aliases.add(wFnDef.getAlias());
        if (wFnDef.isPivotResult()) {
            fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
        } else {
            fieldOIs.add(wFnDef.getOI());
        }
    }
    PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
    tFn.setWdwProcessingOutputOI(wdwOutOI);
    RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
    ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
    wdwTFnDef.setOutputShape(wdwOutShape);
    tFn.setupOutputOI();
    PTFDeserializer.alterOutputOIForStreaming(ptfDesc);
    return ptfDesc;
}