Examples with HiveMetaException org.apache.hadoop.hive.metastore.HiveMetaException used on opensource projects

Example 36 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class HiveSchemaTool method checkMetaStoreSkewedColumnsLocation.

private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String skewedColLoc, skewedColIDRange;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
    } else {
        skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
    }
    if (needsQuotedIdentifier) {
        skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" " + " from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk " + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and " + "sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? order by t.\"TBL_ID\" ";
    } else {
        skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION, db.NAME, db.DB_ID from TBLS t, SDS s, DBS db, SKEWED_COL_VALUE_LOC_MAP sk " + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and t.DB_ID = db.DB_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? order by t.TBL_ID ";
    }
    long maxID = 0, minID = 0;
    long rtnSize = 2000;
    try {
        Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(skewedColIDRange);
        if (res.next()) {
            maxID = res.getLong(1);
            minID = res.getLong(2);
        }
        res.close();
        stmt.close();
        PreparedStatement pStmt = conn.prepareStatement(skewedColLoc);
        while (minID <= maxID) {
            pStmt.setLong(1, minID);
            pStmt.setLong(2, minID + rtnSize);
            res = pStmt.executeQuery();
            while (res.next()) {
                String locValue = res.getString(4);
                String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) + ", String list " + res.getString(3);
                if (!checkLocation(entity, locValue, defaultServers)) {
                    numOfInvalid++;
                }
            }
            res.close();
            minID += rtnSize + 1;
        }
        pStmt.close();
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get skewed columns location info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}

Example 37 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class HiveSchemaHelper method getConnectionToMetastore.

/**
 * Get JDBC connection to metastore db
 * @param userName metastore connection username
 * @param password metastore connection password
 * @param url Metastore URL.  If null will be read from config file.
 * @param driver Driver class.  If null will be read from config file.
 * @param printInfo print connection parameters
 * @param conf hive config object
 * @param schema the schema to create the connection for
 * @return metastore connection object
 * @throws org.apache.hadoop.hive.metastore.HiveMetaException
 */
public static Connection getConnectionToMetastore(String userName, String password, String url, String driver, boolean printInfo, Configuration conf, String schema) throws HiveMetaException {
    try {
        url = url == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTURLKEY, conf) : url;
        driver = driver == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
        if (printInfo) {
            logAndPrintToStdout("Metastore connection URL:\t " + url);
            logAndPrintToStdout("Metastore Connection Driver :\t " + driver);
            logAndPrintToStdout("Metastore connection User:\t " + userName);
            if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) {
                logAndPrintToStdout("Metastore connection Password:\t " + password);
            }
        }
        if ((userName == null) || userName.isEmpty()) {
            throw new HiveMetaException("UserName empty ");
        }
        // load required JDBC driver
        Class.forName(driver);
        // Connect using the JDBC URL and user/pass from conf
        Connection conn = DriverManager.getConnection(url, userName, password);
        if (schema != null) {
            conn.setSchema(schema);
        }
        return conn;
    } catch (IOException | SQLException e) {
        throw new HiveMetaException("Failed to get schema version.", e);
    } catch (ClassNotFoundException e) {
        LOG.error("Unable to find driver class", e);
        throw new HiveMetaException("Failed to load driver", e);
    }
}

Example 38 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method checkMetaStoreTableLocation.

private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String tabLoc, tabIDRange;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
    } else {
        tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
    }
    if (needsQuotedIdentifier) {
        tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " + "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW + "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" order by tbl.\"TBL_ID\" ";
    } else {
        tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_TYPE !='" + TableType.VIRTUAL_VIEW + "' and tbl.TBL_ID >= ? and tbl.TBL_ID <= ?  inner join DBS dbt on tbl.DB_ID = dbt.DB_ID order by tbl.TBL_ID";
    }
    long maxID = 0, minID = 0;
    long rtnSize = 2000;
    try {
        Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(tabIDRange);
        if (res.next()) {
            maxID = res.getLong(1);
            minID = res.getLong(2);
        }
        res.close();
        stmt.close();
        PreparedStatement pStmt = conn.prepareStatement(tabLoc);
        while (minID <= maxID) {
            pStmt.setLong(1, minID);
            pStmt.setLong(2, minID + rtnSize);
            res = pStmt.executeQuery();
            while (res.next()) {
                String locValue = res.getString(3);
                String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
                if (!checkLocation(entity, locValue, defaultServers)) {
                    numOfInvalid++;
                }
            }
            res.close();
            minID += rtnSize + 1;
        }
        pStmt.close();
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get Table Location Info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}

Example 39 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method validateSchemaTables.

boolean validateSchemaTables(Connection conn) throws HiveMetaException {
    String version;
    ResultSet rs = null;
    DatabaseMetaData metadata;
    List<String> dbTables = new ArrayList<>();
    List<String> schemaTables = new ArrayList<>();
    List<String> subScripts = new ArrayList<>();
    Connection hmsConn;
    System.out.println("Validating metastore schema tables");
    try {
        version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
    } catch (HiveMetaException he) {
        logAndPrintToError("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
        System.out.println("Failed in schema table validation.");
        LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
        return false;
    }
    // re-open the hms connection
    hmsConn = getConnectionToMetastore(false);
    LOG.debug("Validating tables in the schema for version " + version);
    try {
        metadata = conn.getMetaData();
        String[] types = { "TABLE" };
        rs = metadata.getTables(null, hmsConn.getSchema(), "%", types);
        String table;
        while (rs.next()) {
            table = rs.getString("TABLE_NAME");
            dbTables.add(table.toLowerCase());
            LOG.debug("Found table " + table + " in HMS dbstore");
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + e.getMessage());
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
            // Not a lot you can do here.
            }
        }
    }
    // parse the schema file to determine the tables that are expected to exist
    // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
    String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
    String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(), metaStoreSchemaInfo.generateInitFileName(version)).getPath();
    try {
        LOG.debug("Parsing schema script " + schemaFile);
        subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        while (subScripts.size() > 0) {
            schemaFile = baseDir + "/" + dbType + "/" + subScripts.remove(0);
            LOG.debug("Parsing subscript " + schemaFile);
            subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        }
    } catch (Exception e) {
        logAndPrintToError("Exception in parsing schema file. Cause:" + e.getMessage());
        System.out.println("Failed in schema table validation.");
        return false;
    }
    LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
    LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
    // now diff the lists
    schemaTables.removeAll(dbTables);
    if (schemaTables.size() > 0) {
        Collections.sort(schemaTables);
        logAndPrintToError("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] are missing from the metastore database schema.");
        System.out.println("Failed in schema table validation.");
        return false;
    } else {
        System.out.println("Succeeded in schema table validation.");
        return true;
    }
}

Example 40 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method checkMetaStoreDBLocation.

private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String dbLoc;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt order by dbt.\"DB_ID\" ";
    } else {
        dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt order by dbt.DB_ID";
    }
    try (Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(dbLoc)) {
        while (res.next()) {
            String locValue = res.getString(3);
            String dbName = getNameOrID(res, 2, 1);
            if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
                numOfInvalid++;
            }
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get DB Location Info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}