Search in sources :

Example 21 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method checkMetaStoreTableLocation.

private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String tabLoc, tabIDRange;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
    } else {
        tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
    }
    if (needsQuotedIdentifier) {
        tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " + "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW + "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" order by tbl.\"TBL_ID\" ";
    } else {
        tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_TYPE !='" + TableType.VIRTUAL_VIEW + "' and tbl.TBL_ID >= ? and tbl.TBL_ID <= ?  inner join DBS dbt on tbl.DB_ID = dbt.DB_ID order by tbl.TBL_ID";
    }
    long maxID = 0, minID = 0;
    long rtnSize = 2000;
    try {
        Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(tabIDRange);
        if (res.next()) {
            maxID = res.getLong(1);
            minID = res.getLong(2);
        }
        res.close();
        stmt.close();
        PreparedStatement pStmt = conn.prepareStatement(tabLoc);
        while (minID <= maxID) {
            pStmt.setLong(1, minID);
            pStmt.setLong(2, minID + rtnSize);
            res = pStmt.executeQuery();
            while (res.next()) {
                String locValue = res.getString(3);
                String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
                if (!checkLocation(entity, locValue, defaultServers)) {
                    numOfInvalid++;
                }
            }
            res.close();
            minID += rtnSize + 1;
        }
        pStmt.close();
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get Table Location Info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}
Also used : SQLException(java.sql.SQLException) PreparedStatement(java.sql.PreparedStatement) Statement(java.sql.Statement) ResultSet(java.sql.ResultSet) HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException) PreparedStatement(java.sql.PreparedStatement)

Example 22 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method validateSchemaTables.

boolean validateSchemaTables(Connection conn) throws HiveMetaException {
    String version;
    ResultSet rs = null;
    DatabaseMetaData metadata;
    List<String> dbTables = new ArrayList<>();
    List<String> schemaTables = new ArrayList<>();
    List<String> subScripts = new ArrayList<>();
    Connection hmsConn;
    System.out.println("Validating metastore schema tables");
    try {
        version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
    } catch (HiveMetaException he) {
        logAndPrintToError("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
        System.out.println("Failed in schema table validation.");
        LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
        return false;
    }
    // re-open the hms connection
    hmsConn = getConnectionToMetastore(false);
    LOG.debug("Validating tables in the schema for version " + version);
    try {
        metadata = conn.getMetaData();
        String[] types = { "TABLE" };
        rs = metadata.getTables(null, hmsConn.getSchema(), "%", types);
        String table;
        while (rs.next()) {
            table = rs.getString("TABLE_NAME");
            dbTables.add(table.toLowerCase());
            LOG.debug("Found table " + table + " in HMS dbstore");
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + e.getMessage());
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
            // Not a lot you can do here.
            }
        }
    }
    // parse the schema file to determine the tables that are expected to exist
    // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
    String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
    String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(), metaStoreSchemaInfo.generateInitFileName(version)).getPath();
    try {
        LOG.debug("Parsing schema script " + schemaFile);
        subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        while (subScripts.size() > 0) {
            schemaFile = baseDir + "/" + dbType + "/" + subScripts.remove(0);
            LOG.debug("Parsing subscript " + schemaFile);
            subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        }
    } catch (Exception e) {
        logAndPrintToError("Exception in parsing schema file. Cause:" + e.getMessage());
        System.out.println("Failed in schema table validation.");
        return false;
    }
    LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
    LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
    // now diff the lists
    schemaTables.removeAll(dbTables);
    if (schemaTables.size() > 0) {
        Collections.sort(schemaTables);
        logAndPrintToError("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] are missing from the metastore database schema.");
        System.out.println("Failed in schema table validation.");
        return false;
    } else {
        System.out.println("Succeeded in schema table validation.");
        return true;
    }
}
Also used : SQLException(java.sql.SQLException) ResultSet(java.sql.ResultSet) ArrayList(java.util.ArrayList) Connection(java.sql.Connection) HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException) DatabaseMetaData(java.sql.DatabaseMetaData) File(java.io.File) HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException) ParseException(org.apache.commons.cli.ParseException) SQLException(java.sql.SQLException) IOException(java.io.IOException)

Example 23 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method checkMetaStoreDBLocation.

private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String dbLoc;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt order by dbt.\"DB_ID\" ";
    } else {
        dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt order by dbt.DB_ID";
    }
    try (Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(dbLoc)) {
        while (res.next()) {
            String locValue = res.getString(3);
            String dbName = getNameOrID(res, 2, 1);
            if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
                numOfInvalid++;
            }
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get DB Location Info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}
Also used : SQLException(java.sql.SQLException) PreparedStatement(java.sql.PreparedStatement) Statement(java.sql.Statement) ResultSet(java.sql.ResultSet) HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException)

Example 24 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method doCreateUser.

private void doCreateUser() throws HiveMetaException {
    testConnectionToMetastore();
    System.out.println("Starting user creation");
    String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir();
    String protoCreateFile = metaStoreSchemaInfo.getCreateUserScript();
    try {
        File createFile = subUserAndPassword(scriptDir, protoCreateFile);
        System.out.println("Creation script " + createFile.getAbsolutePath());
        if (!dryRun) {
            if ("oracle".equals(dbType))
                oracleCreateUserHack(createFile);
            else
                runSqlLine(createFile.getParent(), createFile.getName());
            System.out.println("User creation completed");
        }
    } catch (IOException e) {
        throw new HiveMetaException("User creation FAILED!" + " Metastore unusable !!", e);
    }
}
Also used : HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException) IOException(java.io.IOException) File(java.io.File)

Example 25 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method checkMetaStoreSkewedColumnsLocation.

private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
    String skewedColLoc, skewedColIDRange;
    boolean isValid = true;
    int numOfInvalid = 0;
    if (needsQuotedIdentifier) {
        skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
    } else {
        skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
    }
    if (needsQuotedIdentifier) {
        skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" " + " from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk " + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and " + "sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? order by t.\"TBL_ID\" ";
    } else {
        skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION, db.NAME, db.DB_ID from TBLS t, SDS s, DBS db, SKEWED_COL_VALUE_LOC_MAP sk " + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and t.DB_ID = db.DB_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? order by t.TBL_ID ";
    }
    long maxID = 0, minID = 0;
    long rtnSize = 2000;
    try {
        Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(skewedColIDRange);
        if (res.next()) {
            maxID = res.getLong(1);
            minID = res.getLong(2);
        }
        res.close();
        stmt.close();
        PreparedStatement pStmt = conn.prepareStatement(skewedColLoc);
        while (minID <= maxID) {
            pStmt.setLong(1, minID);
            pStmt.setLong(2, minID + rtnSize);
            res = pStmt.executeQuery();
            while (res.next()) {
                String locValue = res.getString(4);
                String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) + ", String list " + res.getString(3);
                if (!checkLocation(entity, locValue, defaultServers)) {
                    numOfInvalid++;
                }
            }
            res.close();
            minID += rtnSize + 1;
        }
        pStmt.close();
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to get skewed columns location info.", e);
    }
    if (numOfInvalid > 0) {
        isValid = false;
    }
    return isValid;
}
Also used : SQLException(java.sql.SQLException) PreparedStatement(java.sql.PreparedStatement) Statement(java.sql.Statement) ResultSet(java.sql.ResultSet) HiveMetaException(org.apache.hadoop.hive.metastore.HiveMetaException) PreparedStatement(java.sql.PreparedStatement)

Aggregations

HiveMetaException (org.apache.hadoop.hive.metastore.HiveMetaException)28 SQLException (java.sql.SQLException)21 IOException (java.io.IOException)15 ResultSet (java.sql.ResultSet)14 PreparedStatement (java.sql.PreparedStatement)12 Statement (java.sql.Statement)12 File (java.io.File)5 Connection (java.sql.Connection)4 ParseException (org.apache.commons.cli.ParseException)4 ImmutableMap (com.google.common.collect.ImmutableMap)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 OutputStream (java.io.OutputStream)2 PrintStream (java.io.PrintStream)2 DatabaseMetaData (java.sql.DatabaseMetaData)2 ArrayList (java.util.ArrayList)2 CommandLine (org.apache.commons.cli.CommandLine)2 CommandLineParser (org.apache.commons.cli.CommandLineParser)2 GnuParser (org.apache.commons.cli.GnuParser)2 HelpFormatter (org.apache.commons.cli.HelpFormatter)2 Options (org.apache.commons.cli.Options)2