Examples with StructObjectInspector org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector used on opensource projects

Example 61 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class TestStreaming method deserializeDeltaFileRow.

// Assumes stored data schema = [acid fields],string,int,string
// return array of 6 fields, where the last field has the actual data
private static Object[] deserializeDeltaFileRow(Object row, StructObjectInspector inspector) {
    List<? extends StructField> fields = inspector.getAllStructFieldRefs();
    WritableIntObjectInspector f0ins = (WritableIntObjectInspector) fields.get(0).getFieldObjectInspector();
    WritableLongObjectInspector f1ins = (WritableLongObjectInspector) fields.get(1).getFieldObjectInspector();
    WritableIntObjectInspector f2ins = (WritableIntObjectInspector) fields.get(2).getFieldObjectInspector();
    WritableLongObjectInspector f3ins = (WritableLongObjectInspector) fields.get(3).getFieldObjectInspector();
    WritableLongObjectInspector f4ins = (WritableLongObjectInspector) fields.get(4).getFieldObjectInspector();
    StructObjectInspector f5ins = (StructObjectInspector) fields.get(5).getFieldObjectInspector();
    int f0 = f0ins.get(inspector.getStructFieldData(row, fields.get(0)));
    long f1 = f1ins.get(inspector.getStructFieldData(row, fields.get(1)));
    int f2 = f2ins.get(inspector.getStructFieldData(row, fields.get(2)));
    long f3 = f3ins.get(inspector.getStructFieldData(row, fields.get(3)));
    long f4 = f4ins.get(inspector.getStructFieldData(row, fields.get(4)));
    SampleRec f5 = deserializeInner(inspector.getStructFieldData(row, fields.get(5)), f5ins);
    return new Object[] { f0, f1, f2, f3, f4, f5 };
}

Example 62 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class StatsUtils method getSizeOfComplexTypes.

/**
   * Get the size of complex data types
   * @param conf
   *          - hive conf
   * @param oi
   *          - object inspector
   * @return raw data size
   */
public static long getSizeOfComplexTypes(HiveConf conf, ObjectInspector oi) {
    long result = 0;
    int length = 0;
    int listEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_LIST_NUM_ENTRIES);
    int mapEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_NUM_ENTRIES);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            String colTypeLowerCase = oi.getTypeName().toLowerCase();
            if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForStringOfLength(avgColLen);
            } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForByteArrayOfSize(avgColLen);
            } else {
                result += getAvgColLenOfFixedLengthTypes(colTypeLowerCase);
            }
            break;
        case LIST:
            if (oi instanceof StandardConstantListObjectInspector) {
                // constant list projection of known length
                StandardConstantListObjectInspector scloi = (StandardConstantListObjectInspector) oi;
                length = scloi.getWritableConstantValue().size();
                // check if list elements are primitive or Objects
                ObjectInspector leoi = scloi.getListElementObjectInspector();
                if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) {
                    result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length);
                } else {
                    result += JavaDataModel.get().lengthForObjectArrayOfSize(length);
                }
            } else {
                StandardListObjectInspector sloi = (StandardListObjectInspector) oi;
                // list overhead + (configured number of element in list * size of element)
                long elemSize = getSizeOfComplexTypes(conf, sloi.getListElementObjectInspector());
                result += JavaDataModel.get().arrayList() + (listEntries * elemSize);
            }
            break;
        case MAP:
            if (oi instanceof StandardConstantMapObjectInspector) {
                // constant map projection of known length
                StandardConstantMapObjectInspector scmoi = (StandardConstantMapObjectInspector) oi;
                result += getSizeOfMap(scmoi);
            } else {
                StandardMapObjectInspector smoi = (StandardMapObjectInspector) oi;
                result += getSizeOfComplexTypes(conf, smoi.getMapKeyObjectInspector());
                result += getSizeOfComplexTypes(conf, smoi.getMapValueObjectInspector());
                // hash map overhead
                result += JavaDataModel.get().hashMap(mapEntries);
            }
            break;
        case STRUCT:
            if (oi instanceof StandardConstantStructObjectInspector) {
                // constant map projection of known length
                StandardConstantStructObjectInspector scsoi = (StandardConstantStructObjectInspector) oi;
                result += getSizeOfStruct(scsoi);
            } else {
                StructObjectInspector soi = (StructObjectInspector) oi;
                // add constant object overhead for struct
                result += JavaDataModel.get().object();
                // add constant struct field names references overhead
                result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref();
                for (StructField field : soi.getAllStructFieldRefs()) {
                    result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector());
                }
            }
            break;
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            // add constant object overhead for union
            result += JavaDataModel.get().object();
            // add constant size for unions tags
            result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1();
            for (ObjectInspector foi : uoi.getObjectInspectors()) {
                result += getSizeOfComplexTypes(conf, foi);
            }
            break;
        default:
            break;
    }
    return result;
}

Example 63 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class FileSinkOperator method dpSetup.

/**
   * Set up for dynamic partitioning including a new ObjectInspector for the output row.
   */
private void dpSetup() {
    this.bDynParts = false;
    this.numDynParts = dpCtx.getNumDPCols();
    this.dpColNames = dpCtx.getDPColNames();
    this.maxPartitions = dpCtx.getMaxPartitionsPerNode();
    assert numDynParts == dpColNames.size() : "number of dynamic partitions should be the same as the size of DP mapping";
    if (dpColNames != null && dpColNames.size() > 0) {
        this.bDynParts = true;
        assert inputObjInspectors.length == 1 : "FileSinkOperator should have 1 parent, but it has " + inputObjInspectors.length;
        StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[0];
        this.dpStartCol = Utilities.getDPColOffset(conf);
        this.subSetOI = new SubStructObjectInspector(soi, 0, this.dpStartCol);
        this.dpVals = new ArrayList<String>(numDynParts);
        this.dpWritables = new ArrayList<Object>(numDynParts);
    }
}

Example 64 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class FileSinkOperator method process.

@Override
public void process(Object row, int tag) throws HiveException {
    runTimeNumRows++;
    /* Create list bucketing sub-directory only if stored-as-directories is on. */
    String lbDirName = null;
    lbDirName = (lbCtx == null) ? null : generateListBucketingDirName(row);
    if (!bDynParts && !filesCreated) {
        if (lbDirName != null) {
            FSPaths fsp2 = lookupListBucketingPaths(lbDirName);
        } else {
            createBucketFiles(fsp);
        }
    }
    try {
        updateProgress();
        // if DP is enabled, get the final output writers and prepare the real output row
        assert inputObjInspectors[0].getCategory() == ObjectInspector.Category.STRUCT : "input object inspector is not struct";
        if (bDynParts) {
            // we need to read bucket number which is the last column in value (after partition columns)
            if (conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) {
                numDynParts += 1;
            }
            // copy the DP column values from the input row to dpVals
            dpVals.clear();
            dpWritables.clear();
            ObjectInspectorUtils.partialCopyToStandardObject(dpWritables, row, dpStartCol, numDynParts, (StructObjectInspector) inputObjInspectors[0], ObjectInspectorCopyOption.WRITABLE);
            // pass the null value along to the escaping process to determine what the dir should be
            for (Object o : dpWritables) {
                if (o == null || o.toString().length() == 0) {
                    dpVals.add(dpCtx.getDefaultPartitionName());
                } else {
                    dpVals.add(o.toString());
                }
            }
            String invalidPartitionVal;
            if ((invalidPartitionVal = HiveStringUtils.getPartitionValWithInvalidCharacter(dpVals, dpCtx.getWhiteListPattern())) != null) {
                throw new HiveFatalException("Partition value '" + invalidPartitionVal + "' contains a character not matched by whitelist pattern '" + dpCtx.getWhiteListPattern().toString() + "'.  " + "(configure with " + HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname + ")");
            }
            fpaths = getDynOutPaths(dpVals, lbDirName);
            // use SubStructObjectInspector to serialize the non-partitioning columns in the input row
            recordValue = serializer.serialize(row, subSetOI);
        } else {
            if (lbDirName != null) {
                fpaths = lookupListBucketingPaths(lbDirName);
            } else {
                fpaths = fsp;
            }
            recordValue = serializer.serialize(row, inputObjInspectors[0]);
            // is kept track of in the SerDe)
            if (recordValue == null) {
                return;
            }
        }
        rowOutWriters = fpaths.outWriters;
        // check if all record writers implement statistics. if atleast one RW
        // doesn't implement stats interface we will fallback to conventional way
        // of gathering stats
        isCollectRWStats = areAllTrue(statsFromRecordWriter);
        if (conf.isGatherStats() && !isCollectRWStats) {
            SerDeStats stats = serializer.getSerDeStats();
            if (stats != null) {
                fpaths.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
            }
            fpaths.stat.addToStat(StatsSetupConst.ROW_COUNT, 1);
        }
        if ((++numRows == cntr) && isLogInfoEnabled) {
            cntr = logEveryNRows == 0 ? cntr * 10 : numRows + logEveryNRows;
            if (cntr < 0 || numRows < 0) {
                cntr = 0;
                numRows = 1;
            }
            LOG.info(toString() + ": records written - " + numRows);
        }
        // This should always be 0 for the final result file
        int writerOffset = findWriterOffset(row);
        // pass the row rather than recordValue.
        if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
            rowOutWriters[writerOffset].write(recordValue);
        } else if (conf.getWriteType() == AcidUtils.Operation.INSERT) {
            fpaths.updaters[writerOffset].insert(conf.getTransactionId(), row);
        } else {
            // TODO I suspect we could skip much of the stuff above this in the function in the case
            // of update and delete.  But I don't understand all of the side effects of the above
            // code and don't want to skip over it yet.
            // Find the bucket id, and switch buckets if need to
            ObjectInspector rowInspector = bDynParts ? subSetOI : outputObjInspector;
            Object recId = ((StructObjectInspector) rowInspector).getStructFieldData(row, recIdField);
            int bucketNum = bucketInspector.get(recIdInspector.getStructFieldData(recId, bucketField));
            if (fpaths.acidLastBucket != bucketNum) {
                fpaths.acidLastBucket = bucketNum;
                // Switch files
                fpaths.updaters[conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED) ? 0 : ++fpaths.acidFileOffset] = HiveFileFormatUtils.getAcidRecordUpdater(jc, conf.getTableInfo(), bucketNum, conf, fpaths.outPaths[conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED) ? 0 : fpaths.acidFileOffset], rowInspector, reporter, 0);
                if (isDebugEnabled) {
                    LOG.debug("Created updater for bucket number " + bucketNum + " using file " + fpaths.outPaths[conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED) ? 0 : fpaths.acidFileOffset]);
                }
            }
            if (conf.getWriteType() == AcidUtils.Operation.UPDATE) {
                fpaths.updaters[conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED) ? 0 : fpaths.acidFileOffset].update(conf.getTransactionId(), row);
            } else if (conf.getWriteType() == AcidUtils.Operation.DELETE) {
                fpaths.updaters[conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED) ? 0 : fpaths.acidFileOffset].delete(conf.getTransactionId(), row);
            } else {
                throw new HiveException("Unknown write type " + conf.getWriteType().toString());
            }
        }
    } catch (IOException e) {
        throw new HiveException(e);
    } catch (SerDeException e) {
        throw new HiveException(e);
    }
}

Example 65 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class FileSinkOperator method initializeOp.

@Override
protected void initializeOp(Configuration hconf) throws HiveException {
    super.initializeOp(hconf);
    try {
        this.hconf = hconf;
        filesCreated = false;
        isNativeTable = !conf.getTableInfo().isNonNative();
        isTemporary = conf.isTemporary();
        multiFileSpray = conf.isMultiFileSpray();
        totalFiles = conf.getTotalFiles();
        numFiles = conf.getNumFiles();
        dpCtx = conf.getDynPartCtx();
        lbCtx = conf.getLbCtx();
        fsp = prevFsp = null;
        valToPaths = new HashMap<String, FSPaths>();
        taskId = Utilities.getTaskId(hconf);
        initializeSpecPath();
        fs = specPath.getFileSystem(hconf);
        try {
            createHiveOutputFormat(hconf);
        } catch (HiveException ex) {
            logOutputFormatError(hconf, ex);
            throw ex;
        }
        isCompressed = conf.getCompressed();
        parent = Utilities.toTempPath(conf.getDirName());
        statsFromRecordWriter = new boolean[numFiles];
        serializer = (Serializer) conf.getTableInfo().getDeserializerClass().newInstance();
        serializer.initialize(unsetNestedColumnPaths(hconf), conf.getTableInfo().getProperties());
        outputClass = serializer.getSerializedClass();
        if (isLogInfoEnabled) {
            LOG.info("Using serializer : " + serializer + " and formatter : " + hiveOutputFormat + (isCompressed ? " with compression" : ""));
        }
        // Timeout is chosen to make sure that even if one iteration takes more than
        // half of the script.timeout but less than script.timeout, we will still
        // be able to report progress.
        timeOut = hconf.getInt("mapred.healthChecker.script.timeout", 600000) / 2;
        if (hconf instanceof JobConf) {
            jc = (JobConf) hconf;
        } else {
            // test code path
            jc = new JobConf(hconf);
        }
        if (multiFileSpray) {
            partitionEval = new ExprNodeEvaluator[conf.getPartitionCols().size()];
            int i = 0;
            for (ExprNodeDesc e : conf.getPartitionCols()) {
                partitionEval[i++] = ExprNodeEvaluatorFactory.get(e);
            }
            partitionObjectInspectors = initEvaluators(partitionEval, outputObjInspector);
            prtner = (HivePartitioner<HiveKey, Object>) ReflectionUtils.newInstance(jc.getPartitionerClass(), null);
        }
        if (dpCtx != null) {
            dpSetup();
        }
        if (lbCtx != null) {
            lbSetup();
        }
        if (!bDynParts) {
            fsp = new FSPaths(specPath);
            // createBucketFiles(fsp);
            if (!this.isSkewedStoredAsSubDirectories) {
                // special entry for non-DP case
                valToPaths.put("", fsp);
            }
        }
        final StoragePolicyValue tmpStorage = StoragePolicyValue.lookup(HiveConf.getVar(hconf, HIVE_TEMPORARY_TABLE_STORAGE));
        if (isTemporary && fsp != null && tmpStorage != StoragePolicyValue.DEFAULT) {
            final Path outputPath = fsp.taskOutputTempPath;
            StoragePolicyShim shim = ShimLoader.getHadoopShims().getStoragePolicyShim(fs);
            if (shim != null) {
                // directory creation is otherwise within the writers
                fs.mkdirs(outputPath);
                shim.setStoragePolicy(outputPath, tmpStorage);
            }
        }
        if (conf.getWriteType() == AcidUtils.Operation.UPDATE || conf.getWriteType() == AcidUtils.Operation.DELETE) {
            // ROW__ID is always in the first field
            recIdField = ((StructObjectInspector) outputObjInspector).getAllStructFieldRefs().get(0);
            recIdInspector = (StructObjectInspector) recIdField.getFieldObjectInspector();
            // bucket is the second field in the record id
            bucketField = recIdInspector.getAllStructFieldRefs().get(1);
            bucketInspector = (IntObjectInspector) bucketField.getFieldObjectInspector();
        }
        numRows = 0;
        cntr = 1;
        logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
        statsMap.put(getCounterName(Counter.RECORDS_OUT), row_count);
    } catch (HiveException e) {
        throw e;
    } catch (Exception e) {
        e.printStackTrace();
        throw new HiveException(e);
    }
}