Examples with MPartitionColumnStatistics org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics used on opensource projects

Example 1 with MPartitionColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics in project hive by apache.

the class ObjectStore method getMPartitionColumnStatistics.

private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(Table table, List<String> partNames, List<String> colNames, QueryWrapper queryWrapper) throws NoSuchObjectException, MetaException {
    boolean committed = false;
    try {
        openTransaction();
        // ToDo: we need verify the partition column instead
        try {
            validateTableCols(table, colNames);
        } catch (MetaException me) {
            LOG.warn("The table does not have the same column definition as its partition.");
        }
        Query query = queryWrapper.query = pm.newQuery(MPartitionColumnStatistics.class);
        String paramStr = "java.lang.String t1, java.lang.String t2";
        String filter = "tableName == t1 && dbName == t2 && (";
        Object[] params = new Object[colNames.size() + partNames.size() + 2];
        int i = 0;
        params[i++] = table.getTableName();
        params[i++] = table.getDbName();
        int firstI = i;
        for (String s : partNames) {
            filter += ((i == firstI) ? "" : " || ") + "partitionName == p" + i;
            paramStr += ", java.lang.String p" + i;
            params[i++] = s;
        }
        filter += ") && (";
        firstI = i;
        for (String s : colNames) {
            filter += ((i == firstI) ? "" : " || ") + "colName == c" + i;
            paramStr += ", java.lang.String c" + i;
            params[i++] = s;
        }
        filter += ")";
        query.setFilter(filter);
        query.declareParameters(paramStr);
        query.setOrdering("partitionName ascending");
        @SuppressWarnings("unchecked") List<MPartitionColumnStatistics> result = (List<MPartitionColumnStatistics>) query.executeWithArray(params);
        pm.retrieveAll(result);
        committed = commitTransaction();
        return result;
    } catch (Exception ex) {
        LOG.error("Error retrieving statistics via jdo", ex);
        if (ex instanceof MetaException) {
            throw (MetaException) ex;
        }
        throw new MetaException(ex.getMessage());
    } finally {
        if (!committed) {
            rollbackTransaction();
            return Lists.newArrayList();
        }
    }
}

Example 2 with MPartitionColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics in project hive by apache.

the class StatObjectConverter method convertToMPartitionColumnStatistics.

public static MPartitionColumnStatistics convertToMPartitionColumnStatistics(MPartition partition, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj) throws MetaException, NoSuchObjectException {
    if (statsDesc == null || statsObj == null) {
        return null;
    }
    MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics();
    mColStats.setPartition(partition);
    mColStats.setDbName(statsDesc.getDbName());
    mColStats.setTableName(statsDesc.getTableName());
    mColStats.setPartitionName(statsDesc.getPartName());
    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
    mColStats.setColName(statsObj.getColName());
    mColStats.setColType(statsObj.getColType());
    if (statsObj.getStatsData().isSetBooleanStats()) {
        BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
        mColStats.setBooleanStats(boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
    } else if (statsObj.getStatsData().isSetLongStats()) {
        LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
        mColStats.setLongStats(longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, longStats.isSetBitVectors() ? longStats.getBitVectors() : null, longStats.isSetLowValue() ? longStats.getLowValue() : null, longStats.isSetHighValue() ? longStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDoubleStats()) {
        DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
        mColStats.setDoubleStats(doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null, doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDecimalStats()) {
        DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats();
        String low = decimalStats.isSetLowValue() ? createJdoDecimalString(decimalStats.getLowValue()) : null;
        String high = decimalStats.isSetHighValue() ? createJdoDecimalString(decimalStats.getHighValue()) : null;
        mColStats.setDecimalStats(decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null, low, high);
    } else if (statsObj.getStatsData().isSetStringStats()) {
        StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
        mColStats.setStringStats(stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, stringStats.isSetBitVectors() ? stringStats.getBitVectors() : null, stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetBinaryStats()) {
        BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
        mColStats.setBinaryStats(binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetDateStats()) {
        DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats();
        mColStats.setDateStats(dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null, dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
    }
    return mColStats;
}

Example 3 with MPartitionColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics in project hive by apache.

the class DirectSqlUpdateStat method updatePartitionColumnStatistics.

/**
 * Update the statistics for the given partitions. Add the notification logs also.
 * @return map of partition key to column stats if successful, null otherwise.
 */
public Map<String, Map<String, String>> updatePartitionColumnStatistics(Map<String, ColumnStatistics> partColStatsMap, Table tbl, long csId, String validWriteIds, long writeId, List<TransactionalMetaStoreEventListener> transactionalListeners) throws MetaException {
    JDOConnection jdoConn = null;
    Connection dbConn = null;
    boolean committed = false;
    try {
        lockInternal();
        jdoConn = pm.getDataStoreConnection();
        dbConn = (Connection) (jdoConn.getNativeConnection());
        setAnsiQuotes(dbConn);
        Map<PartitionInfo, ColumnStatistics> partitionInfoMap = getPartitionInfo(dbConn, tbl.getId(), partColStatsMap);
        Map<String, Map<String, String>> result = updatePartitionParamTable(dbConn, partitionInfoMap, validWriteIds, writeId, TxnUtils.isAcidTable(tbl));
        Map<PartColNameInfo, MPartitionColumnStatistics> insertMap = new HashMap<>();
        Map<PartColNameInfo, MPartitionColumnStatistics> updateMap = new HashMap<>();
        populateInsertUpdateMap(partitionInfoMap, updateMap, insertMap, dbConn);
        LOG.info("Number of stats to insert  " + insertMap.size() + " update " + updateMap.size());
        if (insertMap.size() != 0) {
            insertIntoPartColStatTable(insertMap, csId, dbConn);
        }
        if (updateMap.size() != 0) {
            updatePartColStatTable(updateMap, dbConn);
        }
        if (transactionalListeners != null) {
            UpdatePartitionColumnStatEventBatch eventBatch = new UpdatePartitionColumnStatEventBatch(null);
            for (Map.Entry entry : result.entrySet()) {
                Map<String, String> parameters = (Map<String, String>) entry.getValue();
                ColumnStatistics colStats = partColStatsMap.get(entry.getKey());
                List<String> partVals = getPartValsFromName(tbl, colStats.getStatsDesc().getPartName());
                UpdatePartitionColumnStatEvent event = new UpdatePartitionColumnStatEvent(colStats, partVals, parameters, tbl, writeId, null);
                eventBatch.addPartColStatEvent(event);
            }
            MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.UPDATE_PARTITION_COLUMN_STAT_BATCH, eventBatch, dbConn, sqlGenerator);
        }
        dbConn.commit();
        committed = true;
        return result;
    } catch (Exception e) {
        LOG.error("Unable to update Column stats for  " + tbl.getTableName(), e);
        throw new MetaException("Unable to update Column stats for  " + tbl.getTableName() + " due to: " + e.getMessage());
    } finally {
        if (!committed) {
            rollbackDBConn(dbConn);
        }
        closeDbConn(jdoConn);
        unlockInternal();
    }
}

Example 4 with MPartitionColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics in project hive by apache.

the class StatObjectConverter method convertToMPartitionColumnStatistics.

public static MPartitionColumnStatistics convertToMPartitionColumnStatistics(MPartition partition, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj, String engine) throws MetaException, NoSuchObjectException {
    if (statsDesc == null || statsObj == null) {
        return null;
    }
    MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics();
    mColStats.setPartition(partition);
    mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME);
    mColStats.setDbName(statsDesc.getDbName());
    mColStats.setTableName(statsDesc.getTableName());
    mColStats.setPartitionName(statsDesc.getPartName());
    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
    mColStats.setColName(statsObj.getColName());
    mColStats.setColType(statsObj.getColType());
    if (statsObj.getStatsData().isSetBooleanStats()) {
        BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
        mColStats.setBooleanStats(boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null);
    } else if (statsObj.getStatsData().isSetLongStats()) {
        LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
        mColStats.setLongStats(longStats.isSetNumNulls() ? longStats.getNumNulls() : null, longStats.isSetNumDVs() ? longStats.getNumDVs() : null, longStats.isSetBitVectors() ? longStats.getBitVectors() : null, longStats.isSetLowValue() ? longStats.getLowValue() : null, longStats.isSetHighValue() ? longStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDoubleStats()) {
        DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
        mColStats.setDoubleStats(doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null, doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
    } else if (statsObj.getStatsData().isSetDecimalStats()) {
        DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats();
        String low = decimalStats.isSetLowValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getLowValue()) : null;
        String high = decimalStats.isSetHighValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getHighValue()) : null;
        mColStats.setDecimalStats(decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null, low, high);
    } else if (statsObj.getStatsData().isSetStringStats()) {
        StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
        mColStats.setStringStats(stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, stringStats.isSetBitVectors() ? stringStats.getBitVectors() : null, stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetBinaryStats()) {
        BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
        mColStats.setBinaryStats(binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null);
    } else if (statsObj.getStatsData().isSetDateStats()) {
        DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats();
        mColStats.setDateStats(dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null, dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
    } else if (statsObj.getStatsData().isSetTimestampStats()) {
        TimestampColumnStatsData timestampStats = statsObj.getStatsData().getTimestampStats();
        mColStats.setTimestampStats(timestampStats.isSetNumNulls() ? timestampStats.getNumNulls() : null, timestampStats.isSetNumDVs() ? timestampStats.getNumDVs() : null, timestampStats.isSetBitVectors() ? timestampStats.getBitVectors() : null, timestampStats.isSetLowValue() ? timestampStats.getLowValue().getSecondsSinceEpoch() : null, timestampStats.isSetHighValue() ? timestampStats.getHighValue().getSecondsSinceEpoch() : null);
    }
    mColStats.setEngine(engine);
    return mColStats;
}

Example 5 with MPartitionColumnStatistics

use of org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics in project hive by apache.

the class ObjectStore method updatePartitionColumnStatistics.

@Override
public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals, String validWriteIds, long writeId) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
    boolean committed = false;
    try {
        openTransaction();
        List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
        ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
        String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
        MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName());
        Table table = convertToTable(mTable);
        Partition partition = convertToPart(getMPartition(catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable), false);
        List<String> colNames = new ArrayList<>();
        for (ColumnStatisticsObj statsObj : statsObjs) {
            colNames.add(statsObj.getColName());
        }
        Map<String, MPartitionColumnStatistics> oldStats = getPartitionColStats(table, statsDesc.getPartName(), colNames, colStats.getEngine());
        MPartition mPartition = getMPartition(catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable);
        if (partition == null) {
            throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
        }
        for (ColumnStatisticsObj statsObj : statsObjs) {
            MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj, colStats.getEngine());
            writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName()));
        }
        // TODO: (HIVE-20109) the col stats stats should be in colstats, not in the partition!
        Map<String, String> newParams = new HashMap<>(mPartition.getParameters());
        StatsSetupConst.setColumnStatsState(newParams, colNames);
        boolean isTxn = TxnUtils.isTransactionalTable(table);
        if (isTxn) {
            if (!areTxnStatsSupported) {
                StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
            } else {
                String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(), newParams, writeId, validWriteIds, true);
                if (errorMsg != null) {
                    throw new MetaException(errorMsg);
                }
                if (!isCurrentStatsValidForTheQuery(mPartition, validWriteIds, true)) {
                    // Make sure we set the flag to invalid regardless of the current value.
                    StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
                    LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition " + statsDesc.getDbName() + "." + statsDesc.getTableName() + "." + statsDesc.getPartName());
                }
                mPartition.setWriteId(writeId);
            }
        }
        mPartition.setParameters(newParams);
        committed = commitTransaction();
        // TODO: what is the "return committed;" about? would it ever return false without throwing?
        return committed ? newParams : null;
    } finally {
        if (!committed) {
            rollbackTransaction();
        }
    }
}