use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.
the class MetastoreSchemaTool method checkMetaStoreTableLocation.
private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
String tabLoc, tabIDRange;
boolean isValid = true;
int numOfInvalid = 0;
if (needsQuotedIdentifier) {
tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
} else {
tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
}
if (needsQuotedIdentifier) {
tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " + "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW + "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" order by tbl.\"TBL_ID\" ";
} else {
tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_TYPE !='" + TableType.VIRTUAL_VIEW + "' and tbl.TBL_ID >= ? and tbl.TBL_ID <= ? inner join DBS dbt on tbl.DB_ID = dbt.DB_ID order by tbl.TBL_ID";
}
long maxID = 0, minID = 0;
long rtnSize = 2000;
try {
Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(tabIDRange);
if (res.next()) {
maxID = res.getLong(1);
minID = res.getLong(2);
}
res.close();
stmt.close();
PreparedStatement pStmt = conn.prepareStatement(tabLoc);
while (minID <= maxID) {
pStmt.setLong(1, minID);
pStmt.setLong(2, minID + rtnSize);
res = pStmt.executeQuery();
while (res.next()) {
String locValue = res.getString(3);
String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
if (!checkLocation(entity, locValue, defaultServers)) {
numOfInvalid++;
}
}
res.close();
minID += rtnSize + 1;
}
pStmt.close();
} catch (SQLException e) {
throw new HiveMetaException("Failed to get Table Location Info.", e);
}
if (numOfInvalid > 0) {
isValid = false;
}
return isValid;
}
use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.
the class MetastoreSchemaTool method validateSchemaTables.
boolean validateSchemaTables(Connection conn) throws HiveMetaException {
String version;
ResultSet rs = null;
DatabaseMetaData metadata;
List<String> dbTables = new ArrayList<>();
List<String> schemaTables = new ArrayList<>();
List<String> subScripts = new ArrayList<>();
Connection hmsConn;
System.out.println("Validating metastore schema tables");
try {
version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
} catch (HiveMetaException he) {
logAndPrintToError("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
System.out.println("Failed in schema table validation.");
LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage());
return false;
}
// re-open the hms connection
hmsConn = getConnectionToMetastore(false);
LOG.debug("Validating tables in the schema for version " + version);
try {
metadata = conn.getMetaData();
String[] types = { "TABLE" };
rs = metadata.getTables(null, hmsConn.getSchema(), "%", types);
String table;
while (rs.next()) {
table = rs.getString("TABLE_NAME");
dbTables.add(table.toLowerCase());
LOG.debug("Found table " + table + " in HMS dbstore");
}
} catch (SQLException e) {
throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + e.getMessage());
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
// Not a lot you can do here.
}
}
}
// parse the schema file to determine the tables that are expected to exist
// we are using oracle schema because it is simpler to parse, no quotes or backticks etc
String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(), metaStoreSchemaInfo.generateInitFileName(version)).getPath();
try {
LOG.debug("Parsing schema script " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
while (subScripts.size() > 0) {
schemaFile = baseDir + "/" + dbType + "/" + subScripts.remove(0);
LOG.debug("Parsing subscript " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
}
} catch (Exception e) {
logAndPrintToError("Exception in parsing schema file. Cause:" + e.getMessage());
System.out.println("Failed in schema table validation.");
return false;
}
LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
// now diff the lists
schemaTables.removeAll(dbTables);
if (schemaTables.size() > 0) {
Collections.sort(schemaTables);
logAndPrintToError("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] are missing from the metastore database schema.");
System.out.println("Failed in schema table validation.");
return false;
} else {
System.out.println("Succeeded in schema table validation.");
return true;
}
}
use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.
the class MetastoreSchemaTool method checkMetaStoreDBLocation.
private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
String dbLoc;
boolean isValid = true;
int numOfInvalid = 0;
if (needsQuotedIdentifier) {
dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt order by dbt.\"DB_ID\" ";
} else {
dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt order by dbt.DB_ID";
}
try (Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(dbLoc)) {
while (res.next()) {
String locValue = res.getString(3);
String dbName = getNameOrID(res, 2, 1);
if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
numOfInvalid++;
}
}
} catch (SQLException e) {
throw new HiveMetaException("Failed to get DB Location Info.", e);
}
if (numOfInvalid > 0) {
isValid = false;
}
return isValid;
}
use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.
the class MetastoreSchemaTool method doCreateUser.
private void doCreateUser() throws HiveMetaException {
testConnectionToMetastore();
System.out.println("Starting user creation");
String scriptDir = metaStoreSchemaInfo.getMetaStoreScriptDir();
String protoCreateFile = metaStoreSchemaInfo.getCreateUserScript();
try {
File createFile = subUserAndPassword(scriptDir, protoCreateFile);
System.out.println("Creation script " + createFile.getAbsolutePath());
if (!dryRun) {
if ("oracle".equals(dbType))
oracleCreateUserHack(createFile);
else
runSqlLine(createFile.getParent(), createFile.getName());
System.out.println("User creation completed");
}
} catch (IOException e) {
throw new HiveMetaException("User creation FAILED!" + " Metastore unusable !!", e);
}
}
use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.
the class MetastoreSchemaTool method checkMetaStoreSkewedColumnsLocation.
private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
String skewedColLoc, skewedColIDRange;
boolean isValid = true;
int numOfInvalid = 0;
if (needsQuotedIdentifier) {
skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
} else {
skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
}
if (needsQuotedIdentifier) {
skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" " + " from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk " + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and " + "sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? order by t.\"TBL_ID\" ";
} else {
skewedColLoc = "select t.TBL_NAME, t.TBL_ID, sk.STRING_LIST_ID_KID, sk.LOCATION, db.NAME, db.DB_ID from TBLS t, SDS s, DBS db, SKEWED_COL_VALUE_LOC_MAP sk " + "where sk.SD_ID = s.SD_ID and s.SD_ID = t.SD_ID and t.DB_ID = db.DB_ID and sk.STRING_LIST_ID_KID >= ? and sk.STRING_LIST_ID_KID <= ? order by t.TBL_ID ";
}
long maxID = 0, minID = 0;
long rtnSize = 2000;
try {
Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(skewedColIDRange);
if (res.next()) {
maxID = res.getLong(1);
minID = res.getLong(2);
}
res.close();
stmt.close();
PreparedStatement pStmt = conn.prepareStatement(skewedColLoc);
while (minID <= maxID) {
pStmt.setLong(1, minID);
pStmt.setLong(2, minID + rtnSize);
res = pStmt.executeQuery();
while (res.next()) {
String locValue = res.getString(4);
String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) + ", String list " + res.getString(3);
if (!checkLocation(entity, locValue, defaultServers)) {
numOfInvalid++;
}
}
res.close();
minID += rtnSize + 1;
}
pStmt.close();
} catch (SQLException e) {
throw new HiveMetaException("Failed to get skewed columns location info.", e);
}
if (numOfInvalid > 0) {
isValid = false;
}
return isValid;
}
Aggregations