Examples with NotNullConstraint org.apache.hadoop.hive.ql.metadata.NotNullConstraint used on opensource projects

Example 1 with NotNullConstraint

use of org.apache.hadoop.hive.ql.metadata.NotNullConstraint in project hive by apache.

the class DDLTask method describeTable.

/**
 * Write the description of a table to a file.
 *
 * @param db
 *          The database in question.
 * @param descTbl
 *          This is the table we're interested in.
 * @return Returns 0 when execution succeeds and above 0 if it fails.
 * @throws HiveException
 *           Throws this exception if an unexpected error occurs.
 * @throws MetaException
 */
private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException, MetaException {
    String colPath = descTbl.getColumnPath();
    String tableName = descTbl.getTableName();
    // describe the table - populate the output stream
    Table tbl = db.getTable(tableName, false);
    if (tbl == null) {
        throw new HiveException(ErrorMsg.INVALID_TABLE, tableName);
    }
    Partition part = null;
    if (descTbl.getPartSpec() != null) {
        part = db.getPartition(tbl, descTbl.getPartSpec(), false);
        if (part == null) {
            throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(descTbl.getPartSpec().keySet(), ','), tableName);
        }
        tbl = part.getTable();
    }
    DataOutputStream outStream = getOutputStream(descTbl.getResFile());
    try {
        LOG.debug("DDLTask: got data for {}", tableName);
        List<FieldSchema> cols = null;
        List<ColumnStatisticsObj> colStats = null;
        Deserializer deserializer = tbl.getDeserializer(true);
        if (deserializer instanceof AbstractSerDe) {
            String errorMsgs = ((AbstractSerDe) deserializer).getConfigurationErrors();
            if (errorMsgs != null && !errorMsgs.isEmpty()) {
                throw new SQLException(errorMsgs);
            }
        }
        if (colPath.equals(tableName)) {
            cols = (part == null || tbl.getTableType() == TableType.VIRTUAL_VIEW) ? tbl.getCols() : part.getCols();
            if (!descTbl.isFormatted()) {
                cols.addAll(tbl.getPartCols());
            }
            if (tbl.isPartitioned() && part == null) {
                // No partitioned specified for partitioned table, lets fetch all.
                Map<String, String> tblProps = tbl.getParameters() == null ? new HashMap<String, String>() : tbl.getParameters();
                Map<String, Long> valueMap = new HashMap<>();
                Map<String, Boolean> stateMap = new HashMap<>();
                for (String stat : StatsSetupConst.supportedStats) {
                    valueMap.put(stat, 0L);
                    stateMap.put(stat, true);
                }
                PartitionIterable parts = new PartitionIterable(db, tbl, null, conf.getIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
                int numParts = 0;
                for (Partition partition : parts) {
                    Map<String, String> props = partition.getParameters();
                    Boolean state = StatsSetupConst.areBasicStatsUptoDate(props);
                    for (String stat : StatsSetupConst.supportedStats) {
                        stateMap.put(stat, stateMap.get(stat) && state);
                        if (props != null && props.get(stat) != null) {
                            valueMap.put(stat, valueMap.get(stat) + Long.parseLong(props.get(stat)));
                        }
                    }
                    numParts++;
                }
                for (String stat : StatsSetupConst.supportedStats) {
                    StatsSetupConst.setBasicStatsState(tblProps, Boolean.toString(stateMap.get(stat)));
                    tblProps.put(stat, valueMap.get(stat).toString());
                }
                tblProps.put(StatsSetupConst.NUM_PARTITIONS, Integer.toString(numParts));
                tbl.setParameters(tblProps);
            }
        } else {
            if (descTbl.isFormatted()) {
                // when column name is specified in describe table DDL, colPath will
                // will be table_name.column_name
                String colName = colPath.split("\\.")[1];
                String[] dbTab = Utilities.getDbTableName(tableName);
                List<String> colNames = new ArrayList<String>();
                colNames.add(colName.toLowerCase());
                if (null == part) {
                    if (tbl.isPartitioned()) {
                        Map<String, String> tblProps = tbl.getParameters() == null ? new HashMap<String, String>() : tbl.getParameters();
                        if (tbl.isPartitionKey(colNames.get(0))) {
                            FieldSchema partCol = tbl.getPartColByName(colNames.get(0));
                            cols = Collections.singletonList(partCol);
                            PartitionIterable parts = new PartitionIterable(db, tbl, null, conf.getIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
                            ColumnInfo ci = new ColumnInfo(partCol.getName(), TypeInfoUtils.getTypeInfoFromTypeString(partCol.getType()), null, false);
                            ColStatistics cs = StatsUtils.getColStatsForPartCol(ci, parts, conf);
                            ColumnStatisticsData data = new ColumnStatisticsData();
                            ColStatistics.Range r = cs.getRange();
                            StatObjectConverter.fillColumnStatisticsData(partCol.getType(), data, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue.toString(), r == null ? null : r.maxValue.toString(), cs.getNumNulls(), cs.getCountDistint(), null, cs.getAvgColLen(), cs.getAvgColLen(), cs.getNumTrues(), cs.getNumFalses());
                            ColumnStatisticsObj cso = new ColumnStatisticsObj(partCol.getName(), partCol.getType(), data);
                            colStats = Collections.singletonList(cso);
                            StatsSetupConst.setColumnStatsState(tblProps, colNames);
                        } else {
                            cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                            List<String> parts = db.getPartitionNames(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), (short) -1);
                            AggrStats aggrStats = db.getAggrColStatsFor(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames, parts);
                            colStats = aggrStats.getColStats();
                            if (parts.size() == aggrStats.getPartsFound()) {
                                StatsSetupConst.setColumnStatsState(tblProps, colNames);
                            } else {
                                StatsSetupConst.removeColumnStatsState(tblProps, colNames);
                            }
                        }
                        tbl.setParameters(tblProps);
                    } else {
                        cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                        colStats = db.getTableColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames);
                    }
                } else {
                    List<String> partitions = new ArrayList<String>();
                    partitions.add(part.getName());
                    cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                    colStats = db.getPartitionColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), partitions, colNames).get(part.getName());
                }
            } else {
                cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
            }
        }
        PrimaryKeyInfo pkInfo = null;
        ForeignKeyInfo fkInfo = null;
        UniqueConstraint ukInfo = null;
        NotNullConstraint nnInfo = null;
        DefaultConstraint dInfo = null;
        CheckConstraint cInfo = null;
        if (descTbl.isExt() || descTbl.isFormatted()) {
            pkInfo = db.getPrimaryKeys(tbl.getDbName(), tbl.getTableName());
            fkInfo = db.getForeignKeys(tbl.getDbName(), tbl.getTableName());
            ukInfo = db.getUniqueConstraints(tbl.getDbName(), tbl.getTableName());
            nnInfo = db.getNotNullConstraints(tbl.getDbName(), tbl.getTableName());
            dInfo = db.getDefaultConstraints(tbl.getDbName(), tbl.getTableName());
            cInfo = db.getCheckConstraints(tbl.getDbName(), tbl.getTableName());
        }
        fixDecimalColumnTypeName(cols);
        // In case the query is served by HiveServer2, don't pad it with spaces,
        // as HiveServer2 output is consumed by JDBC/ODBC clients.
        boolean isOutputPadded = !SessionState.get().isHiveServerQuery();
        formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, descTbl.isFormatted(), descTbl.isExt(), isOutputPadded, colStats, pkInfo, fkInfo, ukInfo, nnInfo, dInfo, cInfo);
        LOG.debug("DDLTask: written data for {}", tableName);
    } catch (SQLException e) {
        throw new HiveException(e, ErrorMsg.GENERIC_ERROR, tableName);
    } finally {
        IOUtils.closeStream(outStream);
    }
    return 0;
}

Example 2 with NotNullConstraint

use of org.apache.hadoop.hive.ql.metadata.NotNullConstraint in project hive by apache.

the class DDLSemanticAnalyzer method hasConstraintsEnabled.

private boolean hasConstraintsEnabled(final String tblName) throws SemanticException {
    NotNullConstraint nnc = null;
    DefaultConstraint dc = null;
    try {
        // retrieve enabled NOT NULL constraint from metastore
        nnc = Hive.get().getEnabledNotNullConstraints(db.getDatabaseCurrent().getName(), tblName);
        dc = Hive.get().getEnabledDefaultConstraints(db.getDatabaseCurrent().getName(), tblName);
    } catch (Exception e) {
        if (e instanceof SemanticException) {
            throw (SemanticException) e;
        } else {
            throw (new RuntimeException(e));
        }
    }
    if ((nnc != null && !nnc.getNotNullConstraints().isEmpty()) || (dc != null && !dc.getDefaultConstraints().isEmpty())) {
        return true;
    }
    return false;
}

Example 3 with NotNullConstraint

use of org.apache.hadoop.hive.ql.metadata.NotNullConstraint in project hive by apache.

the class SemanticAnalyzer method getEnabledNotNullConstraints.

private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException {
    List<Boolean> nullConstraints = new ArrayList<>();
    final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints(tbl.getDbName(), tbl.getTableName());
    ImmutableBitSet bitSet = null;
    if (nnc == null || nnc.getNotNullConstraints().isEmpty()) {
        return bitSet;
    }
    // Build the bitset with not null columns
    ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
    for (String nnCol : nnc.getNotNullConstraints().values()) {
        int nnPos = -1;
        for (int i = 0; i < tbl.getCols().size(); i++) {
            if (tbl.getCols().get(i).getName().equals(nnCol)) {
                nnPos = i;
                builder.set(nnPos);
                break;
            }
        }
    }
    bitSet = builder.build();
    return bitSet;
}

Example 4 with NotNullConstraint

use of org.apache.hadoop.hive.ql.metadata.NotNullConstraint in project hive by apache.

the class ConstraintExprGenerator method getEnabledNotNullConstraints.

private ImmutableBitSet getEnabledNotNullConstraints(Table tbl) throws HiveException {
    final NotNullConstraint nnc = Hive.get().getEnabledNotNullConstraints(tbl.getDbName(), tbl.getTableName());
    if (nnc == null || nnc.getNotNullConstraints().isEmpty()) {
        return null;
    }
    // Build the bitset with not null columns
    ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
    for (String nnCol : nnc.getNotNullConstraints().values()) {
        int nnPos = -1;
        for (int i = 0; i < tbl.getCols().size(); i++) {
            if (tbl.getCols().get(i).getName().equals(nnCol)) {
                nnPos = i;
                builder.set(nnPos);
                break;
            }
        }
    }
    return builder.build();
}

Example 5 with NotNullConstraint

use of org.apache.hadoop.hive.ql.metadata.NotNullConstraint in project hive by apache.

the class AbstractAlterTablePropertiesAnalyzer method hasConstraintsEnabled.

private boolean hasConstraintsEnabled(String tableName) throws SemanticException {
    NotNullConstraint notNullConstriant = null;
    DefaultConstraint defaultConstraint = null;
    try {
        // retrieve enabled NOT NULL constraint from metastore
        notNullConstriant = Hive.get().getEnabledNotNullConstraints(db.getDatabaseCurrent().getName(), tableName);
        defaultConstraint = Hive.get().getEnabledDefaultConstraints(db.getDatabaseCurrent().getName(), tableName);
    } catch (Exception e) {
        if (e instanceof SemanticException) {
            throw (SemanticException) e;
        } else {
            throw (new RuntimeException(e));
        }
    }
    return (notNullConstriant != null && !notNullConstriant.getNotNullConstraints().isEmpty()) || (defaultConstraint != null && !defaultConstraint.getDefaultConstraints().isEmpty());
}