Examples with ColumnStatisticsDesc org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc used on opensource projects

Example 71 with ColumnStatisticsDesc

use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc in project hive by apache.

the class SessionHiveMetaStoreClient method setPartitionColumnStatistics.

/**
 * {@inheritDoc}
 */
@Override
public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws NoSuchObjectException, InvalidObjectException, MetaException, TException, InvalidInputException {
    if (request.getColStatsSize() == 1) {
        ColumnStatistics colStats = request.getColStatsIterator().next();
        ColumnStatisticsDesc desc = colStats.getStatsDesc();
        String dbName = desc.getDbName().toLowerCase();
        String tableName = desc.getTableName().toLowerCase();
        if (getTempTable(dbName, tableName) != null) {
            return updateTempTableColumnStats(dbName, tableName, colStats);
        }
    }
    return super.setPartitionColumnStatistics(request);
}

Example 72 with ColumnStatisticsDesc

use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc in project hive by apache.

the class ColStatsProcessor method constructColumnStatsFromPackedRows.

private List<ColumnStatistics> constructColumnStatsFromPackedRows(Table tbl1) throws HiveException, MetaException, IOException {
    Table tbl = tbl1;
    String partName = null;
    List<String> colName = colStatDesc.getColName();
    List<String> colType = colStatDesc.getColType();
    boolean isTblLevel = colStatDesc.isTblLevel();
    List<ColumnStatistics> stats = new ArrayList<ColumnStatistics>();
    InspectableObject packedRow;
    while ((packedRow = ftOp.getNextRow()) != null) {
        if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) {
            throw new HiveException("Unexpected object type encountered while unpacking row");
        }
        List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();
        StructObjectInspector soi = (StructObjectInspector) packedRow.oi;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        List<Object> list = soi.getStructFieldsDataAsList(packedRow.o);
        List<FieldSchema> partColSchema = tbl.getPartCols();
        // Partition columns are appended at end, we only care about stats column
        int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size();
        assert list != null;
        for (int i = 0; i < numOfStatCols; i++) {
            StructField structField = fields.get(i);
            String columnName = colName.get(i);
            String columnType = colType.get(i);
            Object values = list.get(i);
            try {
                ColumnStatisticsObj statObj = ColumnStatisticsObjTranslator.readHiveStruct(columnName, columnType, structField, values);
                statsObjs.add(statObj);
            } catch (Exception e) {
                if (isStatsReliable) {
                    throw new HiveException("Statistics collection failed while (hive.stats.reliable)", e);
                } else {
                    LOG.debug("Because {} is infinite or NaN, we skip stats.", columnName, e);
                }
            }
        }
        if (!statsObjs.isEmpty()) {
            if (!isTblLevel) {
                List<String> partVals = new ArrayList<String>();
                // Iterate over partition columns to figure out partition name
                for (int i = fields.size() - partColSchema.size(); i < fields.size(); i++) {
                    Object partVal = ((PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(list.get(i));
                    partVals.add(// could be null for default partition
                    partVal == null ? this.conf.getVar(ConfVars.DEFAULTPARTITIONNAME) : partVal.toString());
                }
                partName = Warehouse.makePartName(partColSchema, partVals);
            }
            ColumnStatisticsDesc statsDesc = buildColumnStatsDesc(tbl, partName, isTblLevel);
            ColumnStatistics colStats = new ColumnStatistics();
            colStats.setStatsDesc(statsDesc);
            colStats.setStatsObj(statsObjs);
            stats.add(colStats);
        }
    }
    ftOp.clearFetchContext();
    return stats;
}

Example 73 with ColumnStatisticsDesc

use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc in project hive by apache.

the class ColStatsProcessor method buildColumnStatsDesc.

private ColumnStatisticsDesc buildColumnStatsDesc(Table table, String partName, boolean isTblLevel) {
    String dbName = table.getDbName();
    assert dbName != null;
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setDbName(dbName);
    statsDesc.setTableName(table.getTableName());
    statsDesc.setIsTblLevel(isTblLevel);
    if (!isTblLevel) {
        statsDesc.setPartName(partName);
    } else {
        statsDesc.setPartName(null);
    }
    return statsDesc;
}

Example 74 with ColumnStatisticsDesc

use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc in project hive by apache.

the class MetaStoreDirectSql method getPartitionStats.

public List<ColumnStatistics> getPartitionStats(final String dbName, final String tableName, final List<String> partNames, List<String> colNames, boolean enableBitVector) throws MetaException {
    if (colNames.isEmpty() || partNames.isEmpty()) {
        return Collections.emptyList();
    }
    final boolean doTrace = LOG.isDebugEnabled();
    final String queryText0 = "select \"PARTITION_NAME\", " + getStatsList(enableBitVector) + " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\"" + "  in (%1$s) AND \"PARTITION_NAME\" in (%2$s) order by \"PARTITION_NAME\"";
    Batchable<String, Object[]> b = new Batchable<String, Object[]>() {

        @Override
        public List<Object[]> run(final List<String> inputColNames) throws MetaException {
            Batchable<String, Object[]> b2 = new Batchable<String, Object[]>() {

                @Override
                public List<Object[]> run(List<String> inputPartNames) throws MetaException {
                    String queryText = String.format(queryText0, makeParams(inputColNames.size()), makeParams(inputPartNames.size()));
                    long start = doTrace ? System.nanoTime() : 0;
                    Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
                    Object qResult = executeWithArray(query, prepareParams(dbName, tableName, inputPartNames, inputColNames), queryText);
                    timingTrace(doTrace, queryText0, start, (doTrace ? System.nanoTime() : 0));
                    if (qResult == null) {
                        query.closeAll();
                        return Collections.emptyList();
                    }
                    addQueryAfterUse(query);
                    return ensureList(qResult);
                }
            };
            try {
                return runBatched(partNames, b2);
            } finally {
                addQueryAfterUse(b2);
            }
        }
    };
    List<Object[]> list = runBatched(colNames, b);
    List<ColumnStatistics> result = new ArrayList<ColumnStatistics>(Math.min(list.size(), partNames.size()));
    String lastPartName = null;
    int from = 0;
    for (int i = 0; i <= list.size(); ++i) {
        boolean isLast = i == list.size();
        String partName = isLast ? null : (String) list.get(i)[0];
        if (!isLast && partName.equals(lastPartName)) {
            continue;
        } else if (from != i) {
            ColumnStatisticsDesc csd = new ColumnStatisticsDesc(false, dbName, tableName);
            csd.setPartName(lastPartName);
            result.add(makeColumnStats(list.subList(from, i), csd, 1));
        }
        lastPartName = partName;
        from = i;
        Deadline.checkTimeout();
    }
    b.closeAllQueries();
    return result;
}

Example 75 with ColumnStatisticsDesc

use of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc in project hive by apache.

the class StatObjectConverter method getTableColumnStatisticsDesc.

public static ColumnStatisticsDesc getTableColumnStatisticsDesc(MTableColumnStatistics mStatsObj) {
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setIsTblLevel(true);
    statsDesc.setDbName(mStatsObj.getDbName());
    statsDesc.setTableName(mStatsObj.getTableName());
    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
    return statsDesc;
}