Examples with SelectDesc org.apache.hadoop.hive.ql.plan.SelectDesc used on opensource projects

Example 26 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class TestColumnPrunerProcCtx method buildSelectOperator.

private SelectOperator buildSelectOperator(List<ExprNodeDesc> colList, List<FieldNode> outputCols) {
    SelectOperator selectOperator = mock(SelectOperator.class);
    SelectDesc selectDesc = new SelectDesc(colList, ColumnPrunerProcCtx.toColumnNames(outputCols));
    selectDesc.setSelStarNoCompute(false);
    when(selectOperator.getConf()).thenReturn(selectDesc);
    return selectOperator;
}

Example 27 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class OperatorHealthCheckerHook method checkSelectOperator.

private static void checkSelectOperator(SelectOperator op) {
    SelectDesc conf = op.getConf();
    RowSchema schema = op.getSchema();
    if (schema == null) {
        throw new RuntimeException("I expect a schema for all SelectOp" + op);
    }
    Set<String> cn = new HashSet<>(schema.getColumnNames());
    Set<String> ocn = new HashSet<>(conf.getOutputColumnNames());
    Set<String> diff = Sets.symmetricDifference(cn, ocn);
    if (!diff.isEmpty()) {
        throw new RuntimeException("SelOp output/schema mismatch ");
    }
}

Example 28 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class TestOperators method testScriptOperator.

@Test
public void testScriptOperator() throws Throwable {
    try {
        System.out.println("Testing Script Operator");
        // col1
        ExprNodeDesc exprDesc1 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "col1", "", false);
        // col2
        ExprNodeDesc expr1 = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "col0", "", false);
        ExprNodeDesc expr2 = new ExprNodeConstantDesc("1");
        ExprNodeDesc exprDesc2 = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor().getFuncExprNodeDesc("concat", expr1, expr2);
        // select operator to project these two columns
        ArrayList<ExprNodeDesc> earr = new ArrayList<ExprNodeDesc>();
        earr.add(exprDesc1);
        earr.add(exprDesc2);
        ArrayList<String> outputCols = new ArrayList<String>();
        for (int i = 0; i < earr.size(); i++) {
            outputCols.add("_col" + i);
        }
        SelectDesc selectCtx = new SelectDesc(earr, outputCols);
        Operator<SelectDesc> op = OperatorFactory.get(new CompilationOpContext(), SelectDesc.class);
        op.setConf(selectCtx);
        // scriptOperator to echo the output of the select
        TableDesc scriptOutput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
        TableDesc scriptInput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
        ScriptDesc sd = new ScriptDesc("cat", scriptOutput, TextRecordWriter.class, scriptInput, TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key"));
        Operator<ScriptDesc> sop = OperatorFactory.getAndMakeChild(sd, op);
        // Collect operator to observe the output of the script
        CollectDesc cd = new CollectDesc(Integer.valueOf(10));
        CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, sop);
        op.initialize(new JobConf(TestOperators.class), new ObjectInspector[] { r[0].oi });
        // evaluate on row
        for (int i = 0; i < 5; i++) {
            op.process(r[i].o, 0);
        }
        op.close(false);
        InspectableObject io = new InspectableObject();
        for (int i = 0; i < 5; i++) {
            cdop.retrieve(io);
            System.out.println("[" + i + "] io.o=" + io.o);
            System.out.println("[" + i + "] io.oi=" + io.oi);
            StructObjectInspector soi = (StructObjectInspector) io.oi;
            assert (soi != null);
            StructField a = soi.getStructFieldRef("a");
            StructField b = soi.getStructFieldRef("b");
            assertEquals("" + (i + 1), ((PrimitiveObjectInspector) a.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, a)));
            assertEquals((i) + "1", ((PrimitiveObjectInspector) b.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, b)));
        }
        System.out.println("Script Operator ok");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 29 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class TestOperators method testSetDoneFromChildOperators.

@SuppressWarnings("unchecked")
@Test
public void testSetDoneFromChildOperators() throws HiveException {
    VectorSelectDesc vectorSelectDesc = new VectorSelectDesc();
    vectorSelectDesc.setProjectedOutputColumns(new int[0]);
    vectorSelectDesc.setSelectExpressions(new VectorExpression[0]);
    VectorSelectOperator selOp = new VectorSelectOperator(new CompilationOpContext(), new SelectDesc(), new VectorizationContext("dummy"), vectorSelectDesc);
    VectorSelectOperator childOp = new VectorSelectOperator(new CompilationOpContext(), new SelectDesc(), new VectorizationContext("dummy"), vectorSelectDesc);
    selOp.childOperatorsArray = new Operator[1];
    selOp.childOperatorsArray[0] = childOp;
    selOp.childOperatorsTag = new int[1];
    selOp.childOperatorsTag[0] = 0;
    childOp.childOperatorsArray = new Operator[0];
    Assert.assertFalse(selOp.getDone());
    Assert.assertFalse(childOp.getDone());
    selOp.process(new VectorizedRowBatch(1), 0);
    childOp.setDone(true);
    // selOp is not done, it will detect child's done=true during the next process(batch) call
    Assert.assertFalse(selOp.getDone());
    Assert.assertTrue(childOp.getDone());
    selOp.process(new VectorizedRowBatch(1), 0);
    // selOp detects child's done=true, so it turns to done=true
    Assert.assertTrue(selOp.getDone());
    Assert.assertTrue(childOp.getDone());
}

Example 30 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class BucketingSortingOpProcFactory method extractTraits.

static void extractTraits(BucketingSortingCtx bctx, ReduceSinkOperator rop, Operator<?> childop) throws SemanticException {
    List<ExprNodeDesc> outputValues = Collections.emptyList();
    if (childop instanceof SelectOperator) {
        SelectDesc select = ((SelectOperator) childop).getConf();
        outputValues = ExprNodeDescUtils.backtrack(select.getColList(), childop, rop);
    }
    if (outputValues.isEmpty()) {
        return;
    }
    // Go through the set of partition columns, and find their representatives in the values
    // These represent the bucketed columns
    List<BucketCol> bucketCols = extractBucketCols(rop, outputValues);
    // Go through the set of key columns, and find their representatives in the values
    // These represent the sorted columns
    List<SortCol> sortCols = extractSortCols(rop, outputValues);
    List<ColumnInfo> colInfos = childop.getSchema().getSignature();
    if (!bucketCols.isEmpty()) {
        List<BucketCol> newBucketCols = getNewBucketCols(bucketCols, colInfos);
        bctx.setBucketedCols(childop, newBucketCols);
    }
    if (!sortCols.isEmpty()) {
        List<SortCol> newSortCols = getNewSortCols(sortCols, colInfos);
        bctx.setSortedCols(childop, newSortCols);
    }
}