Examples with CheckConstraint org.apache.hadoop.hive.ql.metadata.CheckConstraint used on opensource projects

Example 1 with CheckConstraint

use of org.apache.hadoop.hive.ql.metadata.CheckConstraint in project hive by apache.

the class DDLTask method describeTable.

/**
 * Write the description of a table to a file.
 *
 * @param db
 *          The database in question.
 * @param descTbl
 *          This is the table we're interested in.
 * @return Returns 0 when execution succeeds and above 0 if it fails.
 * @throws HiveException
 *           Throws this exception if an unexpected error occurs.
 * @throws MetaException
 */
private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException, MetaException {
    String colPath = descTbl.getColumnPath();
    String tableName = descTbl.getTableName();
    // describe the table - populate the output stream
    Table tbl = db.getTable(tableName, false);
    if (tbl == null) {
        throw new HiveException(ErrorMsg.INVALID_TABLE, tableName);
    }
    Partition part = null;
    if (descTbl.getPartSpec() != null) {
        part = db.getPartition(tbl, descTbl.getPartSpec(), false);
        if (part == null) {
            throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(descTbl.getPartSpec().keySet(), ','), tableName);
        }
        tbl = part.getTable();
    }
    DataOutputStream outStream = getOutputStream(descTbl.getResFile());
    try {
        LOG.debug("DDLTask: got data for {}", tableName);
        List<FieldSchema> cols = null;
        List<ColumnStatisticsObj> colStats = null;
        Deserializer deserializer = tbl.getDeserializer(true);
        if (deserializer instanceof AbstractSerDe) {
            String errorMsgs = ((AbstractSerDe) deserializer).getConfigurationErrors();
            if (errorMsgs != null && !errorMsgs.isEmpty()) {
                throw new SQLException(errorMsgs);
            }
        }
        if (colPath.equals(tableName)) {
            cols = (part == null || tbl.getTableType() == TableType.VIRTUAL_VIEW) ? tbl.getCols() : part.getCols();
            if (!descTbl.isFormatted()) {
                cols.addAll(tbl.getPartCols());
            }
            if (tbl.isPartitioned() && part == null) {
                // No partitioned specified for partitioned table, lets fetch all.
                Map<String, String> tblProps = tbl.getParameters() == null ? new HashMap<String, String>() : tbl.getParameters();
                Map<String, Long> valueMap = new HashMap<>();
                Map<String, Boolean> stateMap = new HashMap<>();
                for (String stat : StatsSetupConst.supportedStats) {
                    valueMap.put(stat, 0L);
                    stateMap.put(stat, true);
                }
                PartitionIterable parts = new PartitionIterable(db, tbl, null, conf.getIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
                int numParts = 0;
                for (Partition partition : parts) {
                    Map<String, String> props = partition.getParameters();
                    Boolean state = StatsSetupConst.areBasicStatsUptoDate(props);
                    for (String stat : StatsSetupConst.supportedStats) {
                        stateMap.put(stat, stateMap.get(stat) && state);
                        if (props != null && props.get(stat) != null) {
                            valueMap.put(stat, valueMap.get(stat) + Long.parseLong(props.get(stat)));
                        }
                    }
                    numParts++;
                }
                for (String stat : StatsSetupConst.supportedStats) {
                    StatsSetupConst.setBasicStatsState(tblProps, Boolean.toString(stateMap.get(stat)));
                    tblProps.put(stat, valueMap.get(stat).toString());
                }
                tblProps.put(StatsSetupConst.NUM_PARTITIONS, Integer.toString(numParts));
                tbl.setParameters(tblProps);
            }
        } else {
            if (descTbl.isFormatted()) {
                // when column name is specified in describe table DDL, colPath will
                // will be table_name.column_name
                String colName = colPath.split("\\.")[1];
                String[] dbTab = Utilities.getDbTableName(tableName);
                List<String> colNames = new ArrayList<String>();
                colNames.add(colName.toLowerCase());
                if (null == part) {
                    if (tbl.isPartitioned()) {
                        Map<String, String> tblProps = tbl.getParameters() == null ? new HashMap<String, String>() : tbl.getParameters();
                        if (tbl.isPartitionKey(colNames.get(0))) {
                            FieldSchema partCol = tbl.getPartColByName(colNames.get(0));
                            cols = Collections.singletonList(partCol);
                            PartitionIterable parts = new PartitionIterable(db, tbl, null, conf.getIntVar(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
                            ColumnInfo ci = new ColumnInfo(partCol.getName(), TypeInfoUtils.getTypeInfoFromTypeString(partCol.getType()), null, false);
                            ColStatistics cs = StatsUtils.getColStatsForPartCol(ci, parts, conf);
                            ColumnStatisticsData data = new ColumnStatisticsData();
                            ColStatistics.Range r = cs.getRange();
                            StatObjectConverter.fillColumnStatisticsData(partCol.getType(), data, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue, r == null ? null : r.maxValue, r == null ? null : r.minValue.toString(), r == null ? null : r.maxValue.toString(), cs.getNumNulls(), cs.getCountDistint(), null, cs.getAvgColLen(), cs.getAvgColLen(), cs.getNumTrues(), cs.getNumFalses());
                            ColumnStatisticsObj cso = new ColumnStatisticsObj(partCol.getName(), partCol.getType(), data);
                            colStats = Collections.singletonList(cso);
                            StatsSetupConst.setColumnStatsState(tblProps, colNames);
                        } else {
                            cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                            List<String> parts = db.getPartitionNames(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), (short) -1);
                            AggrStats aggrStats = db.getAggrColStatsFor(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames, parts);
                            colStats = aggrStats.getColStats();
                            if (parts.size() == aggrStats.getPartsFound()) {
                                StatsSetupConst.setColumnStatsState(tblProps, colNames);
                            } else {
                                StatsSetupConst.removeColumnStatsState(tblProps, colNames);
                            }
                        }
                        tbl.setParameters(tblProps);
                    } else {
                        cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                        colStats = db.getTableColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), colNames);
                    }
                } else {
                    List<String> partitions = new ArrayList<String>();
                    partitions.add(part.getName());
                    cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
                    colStats = db.getPartitionColumnStatistics(dbTab[0].toLowerCase(), dbTab[1].toLowerCase(), partitions, colNames).get(part.getName());
                }
            } else {
                cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
            }
        }
        PrimaryKeyInfo pkInfo = null;
        ForeignKeyInfo fkInfo = null;
        UniqueConstraint ukInfo = null;
        NotNullConstraint nnInfo = null;
        DefaultConstraint dInfo = null;
        CheckConstraint cInfo = null;
        if (descTbl.isExt() || descTbl.isFormatted()) {
            pkInfo = db.getPrimaryKeys(tbl.getDbName(), tbl.getTableName());
            fkInfo = db.getForeignKeys(tbl.getDbName(), tbl.getTableName());
            ukInfo = db.getUniqueConstraints(tbl.getDbName(), tbl.getTableName());
            nnInfo = db.getNotNullConstraints(tbl.getDbName(), tbl.getTableName());
            dInfo = db.getDefaultConstraints(tbl.getDbName(), tbl.getTableName());
            cInfo = db.getCheckConstraints(tbl.getDbName(), tbl.getTableName());
        }
        fixDecimalColumnTypeName(cols);
        // In case the query is served by HiveServer2, don't pad it with spaces,
        // as HiveServer2 output is consumed by JDBC/ODBC clients.
        boolean isOutputPadded = !SessionState.get().isHiveServerQuery();
        formatter.describeTable(outStream, colPath, tableName, tbl, part, cols, descTbl.isFormatted(), descTbl.isExt(), isOutputPadded, colStats, pkInfo, fkInfo, ukInfo, nnInfo, dInfo, cInfo);
        LOG.debug("DDLTask: written data for {}", tableName);
    } catch (SQLException e) {
        throw new HiveException(e, ErrorMsg.GENERIC_ERROR, tableName);
    } finally {
        IOUtils.closeStream(outStream);
    }
    return 0;
}

Example 2 with CheckConstraint

use of org.apache.hadoop.hive.ql.metadata.CheckConstraint in project hive by apache.

the class SemanticAnalyzer method getCheckConstraintExpr.

private ExprNodeDesc getCheckConstraintExpr(Table tbl, Operator input, RowResolver inputRR, String dest) throws SemanticException {
    CheckConstraint cc = null;
    try {
        cc = Hive.get().getEnabledCheckConstraints(tbl.getDbName(), tbl.getTableName());
    } catch (HiveException e) {
        throw new SemanticException(e);
    }
    if (cc == null || cc.getCheckConstraints().isEmpty()) {
        return null;
    }
    // build a map which tracks the name of column in input's signature to corresponding table column name
    // this will be used to replace column references in CHECK expression AST with corresponding column name
    // in input
    Map<String, String> col2Cols = new HashMap<>();
    List<ColumnInfo> colInfos = input.getSchema().getSignature();
    int colIdx = 0;
    if (updating(dest)) {
        // if this is an update we need to skip the first col since it is row id
        colIdx = 1;
    }
    for (FieldSchema fs : tbl.getCols()) {
        // since SQL is case insenstive just to make sure that the comparison b/w column names
        // and check expression's column reference work convert the key to lower case
        col2Cols.put(fs.getName().toLowerCase(), colInfos.get(colIdx).getInternalName());
        colIdx++;
    }
    List<String> checkExprStrs = cc.getCheckExpressionList();
    TypeCheckCtx typeCheckCtx = new TypeCheckCtx(inputRR);
    ExprNodeDesc checkAndExprs = null;
    for (String checkExprStr : checkExprStrs) {
        try {
            ParseDriver parseDriver = new ParseDriver();
            ASTNode checkExprAST = parseDriver.parseExpression(checkExprStr);
            // replace column references in checkExprAST with corresponding columns in input
            replaceColumnReference(checkExprAST, col2Cols, inputRR);
            Map<ASTNode, ExprNodeDesc> genExprs = TypeCheckProcFactory.genExprNode(checkExprAST, typeCheckCtx);
            ExprNodeDesc checkExpr = genExprs.get(checkExprAST);
            // Check constraint fails only if it evaluates to false, NULL/UNKNOWN should evaluate to TRUE
            ExprNodeDesc notFalseCheckExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("isnotfalse", checkExpr);
            if (checkAndExprs == null) {
                checkAndExprs = notFalseCheckExpr;
            } else {
                checkAndExprs = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("and", checkAndExprs, notFalseCheckExpr);
            }
        } catch (Exception e) {
            throw new SemanticException(e);
        }
    }
    return checkAndExprs;
}

Example 3 with CheckConstraint

use of org.apache.hadoop.hive.ql.metadata.CheckConstraint in project hive by apache.

the class ConstraintExprGenerator method getCheckConstraintExpr.

private T getCheckConstraintExpr(Table tbl, List<ColumnInfo> inputColInfos, RowResolver inputRR, boolean isUpdateStatement) throws SemanticException {
    CheckConstraint cc = null;
    try {
        cc = Hive.get().getEnabledCheckConstraints(tbl.getDbName(), tbl.getTableName());
    } catch (HiveException e) {
        throw new SemanticException(e);
    }
    if (cc == null || cc.getCheckConstraints().isEmpty()) {
        return null;
    }
    // build a map which tracks the name of column in input's signature to corresponding table column name
    // this will be used to replace column references in CHECK expression AST with corresponding column name
    // in input
    Map<String, String> col2Cols = new HashMap<>();
    int colIdx = 0;
    if (isUpdateStatement) {
        // if this is an update we need to skip the first col since it is row id
        colIdx = 1;
    }
    for (FieldSchema fs : tbl.getCols()) {
        // since SQL is case insenstive just to make sure that the comparison b/w column names
        // and check expression's column reference work convert the key to lower case
        col2Cols.put(fs.getName().toLowerCase(), inputColInfos.get(colIdx).getInternalName());
        colIdx++;
    }
    List<String> checkExprStrs = cc.getCheckExpressionList();
    TypeCheckCtx typeCheckCtx = new TypeCheckCtx(inputRR);
    T checkAndExprs = null;
    for (String checkExprStr : checkExprStrs) {
        try {
            ParseDriver parseDriver = new ParseDriver();
            ASTNode checkExprAST = parseDriver.parseExpression(checkExprStr);
            // replace column references in checkExprAST with corresponding columns in input
            replaceColumnReference(checkExprAST, col2Cols, inputRR);
            Map<ASTNode, T> genExprs = typeCheckProcFactory.genExprNode(checkExprAST, typeCheckCtx);
            T checkExpr = genExprs.get(checkExprAST);
            // Check constraint fails only if it evaluates to false, NULL/UNKNOWN should evaluate to TRUE
            T notFalseCheckExpr = exprProcessor.getFuncExprNodeDesc("isnotfalse", checkExpr);
            if (checkAndExprs == null) {
                checkAndExprs = notFalseCheckExpr;
            } else {
                checkAndExprs = exprProcessor.getFuncExprNodeDesc("and", checkAndExprs, notFalseCheckExpr);
            }
        } catch (Exception e) {
            throw new SemanticException(e);
        }
    }
    return checkAndExprs;
}

Example 4 with CheckConstraint

use of org.apache.hadoop.hive.ql.metadata.CheckConstraint in project hive by apache.

the class DDLPlanUtils method getAlterTableStmtCheckConstraint.

public void getAlterTableStmtCheckConstraint(CheckConstraint ck, List<String> constraints) {
    if (!CheckConstraint.isNotEmpty(ck)) {
        return;
    }
    Map<String, List<CheckConstraint.CheckConstraintCol>> checkConstraints = ck.getCheckConstraints();
    for (String constraintName : checkConstraints.keySet()) {
        List<CheckConstraintCol> checkConstraintCols = checkConstraints.get(constraintName);
        if (checkConstraintCols != null && checkConstraintCols.size() > 0) {
            for (CheckConstraintCol col : checkConstraintCols) {
                ST command = new ST(ALTER_TABLE_ADD_CHECK_CONSTRAINT);
                command.add(DATABASE_NAME, ck.getDatabaseName());
                command.add(TABLE_NAME, ck.getTableName());
                command.add(CONSTRAINT_NAME, constraintName);
                command.add(CHECK_EXPRESSION, col.getCheckExpression());
                command.add(ENABLE, col.getEnable());
                command.add(VALIDATE, col.getValidate());
                command.add(RELY, col.getRely());
                constraints.add(command.render());
            }
        }
    }
}