Examples with SelectDesc org.apache.hadoop.hive.ql.plan.SelectDesc used on opensource projects

Example 41 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class TestVectorSelectOperator method testSelectOperator.

@Test
public void testSelectOperator() throws HiveException {
    List<String> columns = new ArrayList<String>();
    columns.add("a");
    columns.add("b");
    columns.add("c");
    VectorizationContext vc = new VectorizationContext("name", columns);
    SelectDesc selDesc = new SelectDesc(false);
    List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
    ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Long.class, "a", "table", false);
    ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Long.class, "b", "table", false);
    ExprNodeColumnDesc colDesc3 = new ExprNodeColumnDesc(Long.class, "c", "table", false);
    ExprNodeGenericFuncDesc plusDesc = new ExprNodeGenericFuncDesc();
    GenericUDF gudf = new GenericUDFOPPlus();
    plusDesc.setGenericUDF(gudf);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    children.add(colDesc1);
    children.add(colDesc2);
    plusDesc.setChildren(children);
    plusDesc.setTypeInfo(TypeInfoFactory.longTypeInfo);
    colList.add(plusDesc);
    colList.add(colDesc3);
    selDesc.setColList(colList);
    List<String> outputColNames = new ArrayList<String>();
    outputColNames.add("_col0");
    outputColNames.add("_col1");
    selDesc.setOutputColumnNames(outputColNames);
    // CONSIDER unwinding ValidatorVectorSelectOperator as a subclass of VectorSelectOperator.
    VectorSelectDesc vectorSelectDesc = new VectorSelectDesc();
    List<ExprNodeDesc> selectColList = selDesc.getColList();
    VectorExpression[] vectorSelectExprs = new VectorExpression[selectColList.size()];
    for (int i = 0; i < selectColList.size(); i++) {
        ExprNodeDesc expr = selectColList.get(i);
        VectorExpression ve = vc.getVectorExpression(expr);
        vectorSelectExprs[i] = ve;
    }
    vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
    vectorSelectDesc.setProjectedOutputColumns(new int[] { 3, 2 });
    ValidatorVectorSelectOperator vso = new ValidatorVectorSelectOperator(new CompilationOpContext(), selDesc, vc, vectorSelectDesc);
    VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(VectorizedRowBatch.DEFAULT_SIZE, 4, 17);
    vso.process(vrg, 0);
}

Example 42 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class BaseScalarUdfTest method testUdf.

/**
 * This method drives the test. It takes the data from getBaseTable() and
 * feeds it through a SELECT operator with a COLLECT operator after. Each
 * row that is produced by the collect operator is compared to getExpectedResult()
 * and if every row is the expected result the method completes without asserting.
 * @throws HiveException
 */
@Test
public final void testUdf() throws Exception {
    InspectableObject[] data = getBaseTable();
    List<ExprNodeDesc> expressionList = getExpressionList();
    SelectDesc selectCtx = new SelectDesc(expressionList, OperatorTestUtils.createOutputColumnNames(expressionList));
    Operator<SelectDesc> op = OperatorFactory.get(new CompilationOpContext(), SelectDesc.class);
    op.setConf(selectCtx);
    CollectDesc cd = new CollectDesc(Integer.valueOf(10));
    CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, op);
    op.initialize(new JobConf(OperatorTestUtils.class), new ObjectInspector[] { data[0].oi });
    OperatorTestUtils.assertResults(op, cdop, data, getExpectedResult());
}

Example 43 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class HiveTableFunctionScanVisitor method visit.

@Override
OpAttr visit(HiveTableFunctionScan scanRel) throws SemanticException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Translating operator rel#" + scanRel.getId() + ":" + scanRel.getRelTypeName() + " with row type: [" + scanRel.getRowType() + "]");
    }
    RexCall call = (RexCall) scanRel.getCall();
    RowResolver rowResolver = new RowResolver();
    List<String> fieldNames = new ArrayList<>(scanRel.getRowType().getFieldNames());
    List<String> functionFieldNames = new ArrayList<>();
    List<ExprNodeDesc> exprCols = new ArrayList<>();
    Map<String, ExprNodeDesc> colExprMap = new HashMap<>();
    for (int pos = 0; pos < call.getOperands().size(); pos++) {
        ExprNodeConverter converter = new ExprNodeConverter(SemanticAnalyzer.DUMMY_TABLE, fieldNames.get(pos), scanRel.getRowType(), scanRel.getRowType(), ((HiveTableScan) scanRel.getInput(0)).getPartOrVirtualCols(), scanRel.getCluster().getTypeFactory(), true);
        ExprNodeDesc exprCol = call.getOperands().get(pos).accept(converter);
        colExprMap.put(HiveConf.getColumnInternalName(pos), exprCol);
        exprCols.add(exprCol);
        ColumnInfo columnInfo = new ColumnInfo(HiveConf.getColumnInternalName(pos), exprCol.getWritableObjectInspector(), SemanticAnalyzer.DUMMY_TABLE, false);
        rowResolver.put(columnInfo.getTabAlias(), columnInfo.getAlias(), columnInfo);
        functionFieldNames.add(HiveConf.getColumnInternalName(pos));
    }
    OpAttr inputOpAf = hiveOpConverter.dispatch(scanRel.getInputs().get(0));
    TableScanOperator op = (TableScanOperator) inputOpAf.inputs.get(0);
    op.getConf().setRowLimit(1);
    Operator<?> output = OperatorFactory.getAndMakeChild(new SelectDesc(exprCols, functionFieldNames, false), new RowSchema(rowResolver.getRowSchema()), op);
    output.setColumnExprMap(colExprMap);
    Operator<?> funcOp = genUDTFPlan(call, functionFieldNames, output, rowResolver);
    return new OpAttr(null, new HashSet<Integer>(), funcOp);
}

Example 44 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class CorrelationUtilities method replaceReduceSinkWithSelectOperator.

// replace the cRS to SEL operator
protected static SelectOperator replaceReduceSinkWithSelectOperator(ReduceSinkOperator childRS, ParseContext context, AbstractCorrelationProcCtx procCtx) throws SemanticException {
    RowSchema inputRS = childRS.getSchema();
    List<String> columnNames = new ArrayList<String>();
    for (String colName : childRS.getConf().getOutputValueColumnNames()) {
        columnNames.add("VALUE." + colName);
    }
    for (String colName : childRS.getConf().getOutputKeyColumnNames()) {
        columnNames.add("KEY." + colName);
    }
    List<ExprNodeDesc> colExprs = new ArrayList<ExprNodeDesc>();
    colExprs.addAll(childRS.getConf().getKeyCols());
    colExprs.addAll(childRS.getConf().getValueCols());
    SelectDesc select = new SelectDesc(colExprs, columnNames);
    Operator<?> parent = getSingleParent(childRS);
    parent.removeChild(childRS);
    SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild(select, new RowSchema(inputRS.getSignature()), parent);
    sel.setColumnExprMap(childRS.getColumnExprMap());
    sel.setChildOperators(childRS.getChildOperators());
    for (Operator<? extends Serializable> ch : childRS.getChildOperators()) {
        ch.replaceParent(childRS, sel);
    }
    childRS.setChildOperators(null);
    childRS.setParentOperators(null);
    procCtx.addRemovedOperator(childRS);
    return sel;
}

Example 45 with SelectDesc

use of org.apache.hadoop.hive.ql.plan.SelectDesc in project hive by apache.

the class TestExecDriver method populateMapRedPlan6.

@SuppressWarnings("unchecked")
private void populateMapRedPlan6(Table src) throws Exception {
    // map-side work
    ArrayList<String> outputColumns = new ArrayList<String>();
    for (int i = 0; i < 2; i++) {
        outputColumns.add("_col" + i);
    }
    Operator<ReduceSinkDesc> op1 = OperatorFactory.get(ctx, PlanUtils.getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), Utilities.makeList(getStringColumn("tkey"), getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID, NullOrdering.NULLS_LAST));
    Operator<ScriptDesc> op0 = OperatorFactory.get(new ScriptDesc("\'cat\'", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), TextRecordWriter.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "tkey,tvalue"), TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key")), op1);
    Operator<SelectDesc> op4 = OperatorFactory.get(new SelectDesc(Utilities.makeList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0);
    addMapWork(mr, src, "a", op4);
    ReduceWork rWork = new ReduceWork();
    mr.setReduceWork(rWork);
    rWork.setNumReduceTasks(Integer.valueOf(1));
    rWork.setKeyDesc(op1.getConf().getKeySerializeInfo());
    rWork.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
    // reduce side work
    Operator<FileSinkDesc> op3 = OperatorFactory.get(ctx, new FileSinkDesc(new Path(TMPDIR + File.separator + "mapredplan6.out"), Utilities.defaultTd, false));
    Operator<FilterDesc> op2 = OperatorFactory.get(getTestFilterDesc("0"), op3);
    List<ExprNodeDesc> cols = new ArrayList<ExprNodeDesc>();
    cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0));
    cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString() + "." + outputColumns.get(1)));
    Operator<SelectDesc> op5 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op2);
    rWork.setReducer(op5);
}