Examples with MTableColumnStatistics org.apache.hadoop.hive.metastore.model.MTableColumnStatistics used on opensource projects

Example 1 with MTableColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MTableColumnStatistics in project hive by apache.

the class ObjectStore method getPartitionColStats.

/**
   * Get table's column stats
   *
   * @param table
   * @param colNames
   * @return Map of column name and its stats
   * @throws NoSuchObjectException
   * @throws MetaException
   */
private Map<String, MTableColumnStatistics> getPartitionColStats(Table table, List<String> colNames) throws NoSuchObjectException, MetaException {
    Map<String, MTableColumnStatistics> statsMap = Maps.newHashMap();
    QueryWrapper queryWrapper = new QueryWrapper();
    try {
        List<MTableColumnStatistics> stats = getMTableColumnStatistics(table, colNames, queryWrapper);
        for (MTableColumnStatistics cStat : stats) {
            statsMap.put(cStat.getColName(), cStat);
        }
    } finally {
        queryWrapper.close();
    }
    return statsMap;
}

Example 2 with MTableColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MTableColumnStatistics in project hive by apache.

the class ObjectStore method deleteTableColumnStatistics.

@Override
public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
    boolean ret = false;
    Query query = null;
    if (dbName == null) {
        dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
    }
    if (tableName == null) {
        throw new InvalidInputException("Table name is null.");
    }
    try {
        openTransaction();
        MTable mTable = getMTable(dbName, tableName);
        MTableColumnStatistics mStatsObj;
        List<MTableColumnStatistics> mStatsObjColl;
        if (mTable == null) {
            throw new NoSuchObjectException("Table " + tableName + "  for which stats deletion is requested doesn't exist");
        }
        query = pm.newQuery(MTableColumnStatistics.class);
        String filter;
        String parameters;
        if (colName != null) {
            filter = "table.tableName == t1 && dbName == t2 && colName == t3";
            parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3";
        } else {
            filter = "table.tableName == t1 && dbName == t2";
            parameters = "java.lang.String t1, java.lang.String t2";
        }
        query.setFilter(filter);
        query.declareParameters(parameters);
        if (colName != null) {
            query.setUnique(true);
            mStatsObj = (MTableColumnStatistics) query.execute(HiveStringUtils.normalizeIdentifier(tableName), HiveStringUtils.normalizeIdentifier(dbName), HiveStringUtils.normalizeIdentifier(colName));
            pm.retrieve(mStatsObj);
            if (mStatsObj != null) {
                pm.deletePersistent(mStatsObj);
            } else {
                throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName + " col=" + colName);
            }
        } else {
            mStatsObjColl = (List<MTableColumnStatistics>) query.execute(HiveStringUtils.normalizeIdentifier(tableName), HiveStringUtils.normalizeIdentifier(dbName));
            pm.retrieveAll(mStatsObjColl);
            if (mStatsObjColl != null) {
                pm.deletePersistentAll(mStatsObjColl);
            } else {
                throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName);
            }
        }
        ret = commitTransaction();
    } catch (NoSuchObjectException e) {
        rollbackTransaction();
        throw e;
    } finally {
        if (!ret) {
            rollbackTransaction();
        }
        if (query != null) {
            query.closeAll();
        }
    }
    return ret;
}

Example 3 with MTableColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MTableColumnStatistics in project hive by apache.

the class ObjectStore method updateTableColumnStatistics.

@Override
public boolean updateTableColumnStatistics(ColumnStatistics colStats) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
    boolean committed = false;
    openTransaction();
    try {
        List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
        ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
        // DataNucleus objects get detached all over the place for no (real) reason.
        // So let's not use them anywhere unless absolutely necessary.
        Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
        List<String> colNames = new ArrayList<>();
        for (ColumnStatisticsObj statsObj : statsObjs) {
            colNames.add(statsObj.getColName());
        }
        Map<String, MTableColumnStatistics> oldStats = getPartitionColStats(table, colNames);
        for (ColumnStatisticsObj statsObj : statsObjs) {
            // We have to get mtable again because DataNucleus.
            MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj);
            writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName()));
            colNames.add(statsObj.getColName());
        }
        // Set the table properties
        // No need to check again if it exists.
        String dbname = table.getDbName();
        String name = table.getTableName();
        MTable oldt = getMTable(dbname, name);
        Map<String, String> parameters = table.getParameters();
        StatsSetupConst.setColumnStatsState(parameters, colNames);
        oldt.setParameters(parameters);
        committed = commitTransaction();
        return committed;
    } finally {
        if (!committed) {
            rollbackTransaction();
        }
    }
}

Example 4 with MTableColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MTableColumnStatistics in project hive by apache.

the class ObjectStore method getMTableColumnStatistics.

private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, QueryWrapper queryWrapper) throws MetaException {
    if (colNames == null || colNames.isEmpty()) {
        return null;
    }
    boolean committed = false;
    try {
        openTransaction();
        List<MTableColumnStatistics> result = null;
        validateTableCols(table, colNames);
        Query query = queryWrapper.query = pm.newQuery(MTableColumnStatistics.class);
        String filter = "tableName == t1 && dbName == t2 && (";
        String paramStr = "java.lang.String t1, java.lang.String t2";
        Object[] params = new Object[colNames.size() + 2];
        params[0] = table.getTableName();
        params[1] = table.getDbName();
        for (int i = 0; i < colNames.size(); ++i) {
            filter += ((i == 0) ? "" : " || ") + "colName == c" + i;
            paramStr += ", java.lang.String c" + i;
            params[i + 2] = colNames.get(i);
        }
        filter += ")";
        query.setFilter(filter);
        query.declareParameters(paramStr);
        result = (List<MTableColumnStatistics>) query.executeWithArray(params);
        pm.retrieveAll(result);
        if (result.size() > colNames.size()) {
            throw new MetaException("Unexpected " + result.size() + " statistics for " + colNames.size() + " columns");
        }
        committed = commitTransaction();
        return result;
    } catch (Exception ex) {
        LOG.error("Error retrieving statistics via jdo", ex);
        if (ex instanceof MetaException) {
            throw (MetaException) ex;
        }
        throw new MetaException(ex.getMessage());
    } finally {
        if (!committed) {
            rollbackTransaction();
        }
    }
}

Example 5 with MTableColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MTableColumnStatistics in project hive by apache.

the class StatObjectConverter method convertToMTableColumnStatistics.

// JDO
public static MTableColumnStatistics convertToMTableColumnStatistics(MTable table, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj) throws NoSuchObjectException, MetaException, InvalidObjectException {
    if (statsObj == null || statsDesc == null) {
        throw new InvalidObjectException("Invalid column stats object");
    }
    MTableColumnStatistics mColStats = new MTableColumnStatistics();
    mColStats.setTable(table);
    mColStats.setDbName(statsDesc.getDbName());
    mColStats.setTableName(statsDesc.getTableName());
    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
    mColStats.setColName(statsObj.getColName());
    mColStats.setColType(statsObj.getColType());
    if (statsObj.getStatsData().isSetBooleanStats()) {
        BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
        mColStats.setBooleanStats(boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
    } else if (statsObj.getStatsData().isSetLongStats()) {
        LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
        mColStats.setLongStats(longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, longStats.isSetLowValue() ? longStats.getLowValue() : null, longStats.isSetHighValue() ? longStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDoubleStats()) {
        DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
        mColStats.setDoubleStats(doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDecimalStats()) {
        DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats();
        String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null;
        String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null;
        mColStats.setDecimalStats(decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, low, high);
    } else if (statsObj.getStatsData().isSetStringStats()) {
        StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
        mColStats.setStringStats(stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetBinaryStats()) {
        BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
        mColStats.setBinaryStats(binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetDateStats()) {
        DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats();
        mColStats.setDateStats(dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
    }
    return mColStats;
}