Examples with ObjectInspectorFactory.getStandardStructObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector used on opensource projects

Example 76 with ObjectInspectorFactory.getStandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector in project hive by apache.

the class UnionOperator method initializeOp.

/**
 * UnionOperator will transform the input rows if the inputObjInspectors from
 * different parents are different. If one parent has exactly the same
 * ObjectInspector as the output ObjectInspector, then we don't need to do
 * transformation for that parent. This information is recorded in
 * needsTransform[].
 */
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
    super.initializeOp(hconf);
    int parents = parentOperators.size();
    parentObjInspectors = new StructObjectInspector[parents];
    parentFields = new List[parents];
    int columns = 0;
    for (int p = 0; p < parents; p++) {
        parentObjInspectors[p] = (StructObjectInspector) inputObjInspectors[p];
        parentFields[p] = parentObjInspectors[p].getAllStructFieldRefs();
        if (p == 0 || parentFields[p].size() < columns) {
            columns = parentFields[p].size();
        }
    }
    // Get columnNames from the first parent
    ArrayList<String> columnNames = new ArrayList<String>(columns);
    for (int c = 0; c < columns; c++) {
        columnNames.add(parentFields[0].get(c).getFieldName());
    }
    // Get outputFieldOIs
    columnTypeResolvers = new ReturnObjectInspectorResolver[columns];
    for (int c = 0; c < columns; c++) {
        columnTypeResolvers[c] = new ReturnObjectInspectorResolver(true);
    }
    for (int p = 0; p < parents; p++) {
        // When columns is 0, the union operator is empty.
        assert (columns == 0 || parentFields[p].size() == columns);
        for (int c = 0; c < columns; c++) {
            if (!columnTypeResolvers[c].updateForUnionAll(parentFields[p].get(c).getFieldObjectInspector())) {
                // checked in SemanticAnalyzer. Should not happen
                throw new HiveException("Incompatible types for union operator");
            }
        }
    }
    ArrayList<ObjectInspector> outputFieldOIs = new ArrayList<ObjectInspector>(columns);
    for (int c = 0; c < columns; c++) {
        // can be null for void type
        ObjectInspector fieldOI = parentFields[0].get(c).getFieldObjectInspector();
        outputFieldOIs.add(columnTypeResolvers[c].get(fieldOI));
    }
    // create output row ObjectInspector
    outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, outputFieldOIs);
    outputRow = new ArrayList<Object>(columns);
    for (int c = 0; c < columns; c++) {
        outputRow.add(null);
    }
    // whether we need to do transformation for each parent
    needsTransform = new boolean[parents];
    for (int p = 0; p < parents; p++) {
        // Testing using != is good enough, because we use ObjectInspectorFactory
        // to
        // create ObjectInspectors.
        needsTransform[p] = (inputObjInspectors[p] != outputObjInspector);
        if (LOG.isInfoEnabled() && needsTransform[p]) {
            LOG.info("Union Operator needs to transform row from parent[" + p + "] from " + inputObjInspectors[p] + " to " + outputObjInspector);
        }
    }
}

Example 77 with ObjectInspectorFactory.getStandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector in project hive by apache.

the class SemanticAnalyzer method genUDTFPlan.

private Operator genUDTFPlan(GenericUDTF genericUDTF, String outputTableAlias, ArrayList<String> colAliases, QB qb, Operator input, boolean outerLV) throws SemanticException {
    // No GROUP BY / DISTRIBUTE BY / SORT BY / CLUSTER BY
    QBParseInfo qbp = qb.getParseInfo();
    if (!qbp.getDestToGroupBy().isEmpty()) {
        throw new SemanticException(ErrorMsg.UDTF_NO_GROUP_BY.getMsg());
    }
    if (!qbp.getDestToDistributeBy().isEmpty()) {
        throw new SemanticException(ErrorMsg.UDTF_NO_DISTRIBUTE_BY.getMsg());
    }
    if (!qbp.getDestToSortBy().isEmpty()) {
        throw new SemanticException(ErrorMsg.UDTF_NO_SORT_BY.getMsg());
    }
    if (!qbp.getDestToClusterBy().isEmpty()) {
        throw new SemanticException(ErrorMsg.UDTF_NO_CLUSTER_BY.getMsg());
    }
    if (!qbp.getAliasToLateralViews().isEmpty()) {
        throw new SemanticException(ErrorMsg.UDTF_LATERAL_VIEW.getMsg());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Table alias: " + outputTableAlias + " Col aliases: " + colAliases);
    }
    // Use the RowResolver from the input operator to generate a input
    // ObjectInspector that can be used to initialize the UDTF. Then, the
    // resulting output object inspector can be used to make the RowResolver
    // for the UDTF operator
    RowResolver selectRR = opParseCtx.get(input).getRowResolver();
    ArrayList<ColumnInfo> inputCols = selectRR.getColumnInfos();
    // Create the object inspector for the input columns and initialize the UDTF
    ArrayList<String> colNames = new ArrayList<String>();
    ObjectInspector[] colOIs = new ObjectInspector[inputCols.size()];
    for (int i = 0; i < inputCols.size(); i++) {
        colNames.add(inputCols.get(i).getInternalName());
        colOIs[i] = inputCols.get(i).getObjectInspector();
    }
    StandardStructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(colNames, Arrays.asList(colOIs));
    StructObjectInspector outputOI = genericUDTF.initialize(rowOI);
    int numUdtfCols = outputOI.getAllStructFieldRefs().size();
    if (colAliases.isEmpty()) {
        // user did not specfied alias names, infer names from outputOI
        for (StructField field : outputOI.getAllStructFieldRefs()) {
            colAliases.add(field.getFieldName());
        }
    }
    // Make sure that the number of column aliases in the AS clause matches
    // the number of columns output by the UDTF
    int numSuppliedAliases = colAliases.size();
    if (numUdtfCols != numSuppliedAliases) {
        throw new SemanticException(ErrorMsg.UDTF_ALIAS_MISMATCH.getMsg("expected " + numUdtfCols + " aliases " + "but got " + numSuppliedAliases));
    }
    // Generate the output column info's / row resolver using internal names.
    ArrayList<ColumnInfo> udtfCols = new ArrayList<ColumnInfo>();
    Iterator<String> colAliasesIter = colAliases.iterator();
    for (StructField sf : outputOI.getAllStructFieldRefs()) {
        String colAlias = colAliasesIter.next();
        assert (colAlias != null);
        // Since the UDTF operator feeds into a LVJ operator that will rename
        // all the internal names, we can just use field name from the UDTF's OI
        // as the internal name
        ColumnInfo col = new ColumnInfo(sf.getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(sf.getFieldObjectInspector()), outputTableAlias, false);
        udtfCols.add(col);
    }
    // Create the row resolver for this operator from the output columns
    RowResolver out_rwsch = new RowResolver();
    for (int i = 0; i < udtfCols.size(); i++) {
        out_rwsch.put(outputTableAlias, colAliases.get(i), udtfCols.get(i));
    }
    // Add the UDTFOperator to the operator DAG
    Operator<?> udtf = putOpInsertMap(OperatorFactory.getAndMakeChild(new UDTFDesc(genericUDTF, outerLV), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch);
    return udtf;
}

Example 78 with ObjectInspectorFactory.getStandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector in project hive by apache.

the class PartExprEvalUtils method evalExprWithPart.

/**
 * Evaluate expression with partition columns
 *
 * @param expr
 * @param partSpec
 * @param rowObjectInspector
 * @return value returned by the expression
 * @throws HiveException
 */
public static synchronized Object evalExprWithPart(ExprNodeDesc expr, Partition p, List<VirtualColumn> vcs, StructObjectInspector rowObjectInspector) throws HiveException {
    LinkedHashMap<String, String> partSpec = p.getSpec();
    Properties partProps = p.getSchema();
    String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
    String[] partKeyTypes = pcolTypes.trim().split(":");
    if (partSpec.size() != partKeyTypes.length) {
        throw new HiveException("Internal error : Partition Spec size, " + partSpec.size() + " doesn't match partition key definition size, " + partKeyTypes.length);
    }
    boolean hasVC = vcs != null && !vcs.isEmpty();
    Object[] rowWithPart = new Object[hasVC ? 3 : 2];
    // Create the row object
    ArrayList<String> partNames = new ArrayList<String>();
    ArrayList<Object> partValues = new ArrayList<Object>();
    ArrayList<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
    int i = 0;
    for (Map.Entry<String, String> entry : partSpec.entrySet()) {
        partNames.add(entry.getKey());
        ObjectInspector oi = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(partKeyTypes[i++]));
        partValues.add(ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi).convert(entry.getValue()));
        partObjectInspectors.add(oi);
    }
    StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors);
    rowWithPart[1] = partValues;
    ArrayList<StructObjectInspector> ois = new ArrayList<StructObjectInspector>(2);
    ois.add(rowObjectInspector);
    ois.add(partObjectInspector);
    if (hasVC) {
        ois.add(VirtualColumn.getVCSObjectInspector(vcs));
    }
    StructObjectInspector rowWithPartObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(ois);
    ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(expr);
    ObjectInspector evaluateResultOI = evaluator.initialize(rowWithPartObjectInspector);
    Object evaluateResultO = evaluator.evaluate(rowWithPart);
    return ((PrimitiveObjectInspector) evaluateResultOI).getPrimitiveJavaObject(evaluateResultO);
}

Example 79 with ObjectInspectorFactory.getStandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector in project hive by apache.

the class PTFTranslator method getStandardStructOI.

/**
 * For each column on the input RR, construct a StructField for it
 * OI is constructed using the list of input column names and
 * their corresponding OIs.
 *
 * @param rr
 * @return
 */
public static StructObjectInspector getStandardStructOI(RowResolver rr) {
    StructObjectInspector oi;
    ArrayList<ColumnInfo> colLists = rr.getColumnInfos();
    ArrayList<String> structFieldNames = new ArrayList<String>();
    ArrayList<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
    for (ColumnInfo columnInfo : colLists) {
        String colName = columnInfo.getInternalName();
        ObjectInspector colOI = columnInfo.getObjectInspector();
        structFieldNames.add(colName);
        structFieldObjectInspectors.add(colOI);
    }
    oi = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
    return oi;
}

Example 80 with ObjectInspectorFactory.getStandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector in project SQLWindowing by hbutani.

the class TranslateUtils method setupSelectOI.

/**
 * Use the column aliases and OIs on each ColumnDef
 * to recreate the OI on the select list
 * @param sDef
 */
public static void setupSelectOI(SelectDef sDef) {
    ArrayList<ColumnDef> selColDefs = sDef.getColumns();
    ArrayList<String> colAliases = new ArrayList<String>();
    ArrayList<ObjectInspector> colOIs = new ArrayList<ObjectInspector>();
    for (ColumnDef colDef : selColDefs) {
        colAliases.add(colDef.getAlias());
        colOIs.add(colDef.getOI());
    }
    sDef.setOI(ObjectInspectorFactory.getStandardStructObjectInspector(colAliases, colOIs));
}