Examples with HiveMetaException org.apache.hadoop.hive.metastore.HiveMetaException used on opensource projects

Example 51 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class MetastoreSchemaTool method runScript.

@VisibleForTesting
public final int runScript(String[] args, InputStream scriptStream) {
    try {
        init(findHomeDir(), args, null, MetastoreConf.newMetastoreConf());
        // Cannot run script directly from input stream thus copy is necessary.
        File scriptFile = File.createTempFile("schemaToolTmpScript", "sql");
        scriptFile.deleteOnExit();
        FileUtils.copyToFile(scriptStream, scriptFile);
        execSql(scriptFile.getAbsolutePath());
        return 0;
    } catch (HiveMetaException | IOException e) {
        throw new RuntimeException("Failed to run script " + scriptStream, e);
    }
}

Example 52 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class SchemaToolTaskMergeCatalog method execute.

@Override
void execute() throws HiveMetaException {
    if (fromCatalog == null || toCatalog == null) {
        throw new HiveMetaException("Merge catalog requires --mergeCatalog and --toCatalog arguments");
    }
    System.out.println("Merging databases from " + fromCatalog + " to " + toCatalog);
    Connection conn = schemaTool.getConnectionToMetastore(true);
    boolean success = false;
    long initTime, prevTime, curTime;
    try {
        // determine conflicts between catalogs first
        try (Statement stmt = conn.createStatement()) {
            initTime = System.currentTimeMillis();
            // TODO ensure both catalogs exist first.
            // Detect conflicting databases
            String conflicts = String.format(schemaTool.quote(DB_CONFLICTS_STMT), fromCatalog, toCatalog);
            System.out.println("Determining name conflicts between databases across catalogs");
            LOG.info("[DB Conflicts] Executing SQL:" + conflicts);
            ResultSet rs = stmt.executeQuery(conflicts);
            boolean cleanMerge = true;
            while (rs.next()) {
                cleanMerge = false;
                System.out.println("Name conflict(s) between merging catalogs, database " + rs.getString(1) + " exists in catalogs " + rs.getString(2) + " and " + rs.getString(3));
            }
            if (!cleanMerge) {
                System.out.println("[ERROR] Please resolve the database name conflicts shown above manually and retry the mergeCatalog operation.");
                System.exit(1);
            }
            conn.setAutoCommit(false);
            String insert = String.format(schemaTool.quote(ADD_AUTOPURGE_TO_TABLE), "EXTERNAL", "TRUE", "MANAGED_TABLE", fromCatalog);
            System.out.println("Setting external=true on all MANAGED tables in catalog " + fromCatalog);
            LOG.debug("[external table property] Executing SQL:" + insert);
            prevTime = System.currentTimeMillis();
            int count = stmt.executeUpdate(insert);
            curTime = System.currentTimeMillis();
            System.out.println("Set external.table.purge on " + count + " tables, time taken (ms):" + (curTime - prevTime));
            insert = String.format(schemaTool.quote(ADD_AUTOPURGE_TO_TABLE), "external.table.purge", "true", "MANAGED_TABLE", fromCatalog);
            System.out.println("Setting external.table.purge=true on all MANAGED tables in catalog " + fromCatalog);
            LOG.debug("[external.table.purge] Executing SQL:" + insert);
            prevTime = curTime;
            count = stmt.executeUpdate(insert);
            curTime = System.currentTimeMillis();
            System.out.println("Set external.table.purge on " + count + " tables, time taken (ms):" + (curTime - prevTime));
            String update = String.format(schemaTool.quote(CONVERT_TABLE_TO_EXTERNAL), "EXTERNAL_TABLE", "MANAGED_TABLE", fromCatalog);
            System.out.println("Setting tableType to EXTERNAL on all MANAGED tables in catalog " + fromCatalog);
            LOG.debug("[tableType=EXTERNAL_TABLE] Executing SQL:" + update);
            prevTime = curTime;
            count = stmt.executeUpdate(update);
            curTime = System.currentTimeMillis();
            System.out.println("Set tableType=EXTERNAL_TABLE on " + count + " tables, time taken (ms):" + (curTime - prevTime));
            String merge = String.format(schemaTool.quote(MERGE_CATALOG_STMT), toCatalog, fromCatalog);
            System.out.println("Setting catalog names on all databases in catalog " + fromCatalog);
            LOG.debug("[catalog name] Executing SQL:" + merge);
            prevTime = curTime;
            count = stmt.executeUpdate(merge);
            curTime = System.currentTimeMillis();
            System.out.println("Changed catalog names on " + count + " databases, time taken (ms):" + (curTime - prevTime));
            if (count == 0) {
                LOG.info(count + " databases have been merged from catalog " + fromCatalog + " into " + toCatalog);
            }
            if (schemaTool.isDryRun()) {
                conn.rollback();
                success = true;
            } else {
                conn.commit();
                System.out.println("Committed the changes. Total time taken (ms):" + (curTime - initTime));
                success = true;
            }
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to merge catalog", e);
    } finally {
        try {
            if (!success) {
                System.out.println("Rolling back transaction");
                conn.rollback();
            }
            conn.close();
        } catch (SQLException e) {
            // Not really much we can do here.
            LOG.error("Failed to rollback, everything will probably go bad from here.", e);
            try {
                conn.close();
            } catch (SQLException ex) {
                LOG.warn("Failed to close connection.", ex);
            }
        }
    }
}

Example 53 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class SchemaToolTaskValidate method validateColumnNullValues.

@VisibleForTesting
boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
    System.out.println("Validating columns for incorrect NULL values.");
    boolean isValid = true;
    String queryColumnNullValues = schemaTool.quote(QUERY_COLUMN_NULL_VALUES);
    try (Statement stmt = conn.createStatement();
        ResultSet res = stmt.executeQuery(queryColumnNullValues)) {
        while (res.next()) {
            long tableId = res.getLong("TBL_ID");
            String tableName = res.getString("TBL_NAME");
            String tableType = res.getString("TBL_TYPE");
            isValid = false;
            System.err.println("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType);
        }
        System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n");
        return isValid;
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
    }
}

Example 54 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class SchemaToolTaskValidate method validateSchemaTables.

@VisibleForTesting
boolean validateSchemaTables(Connection conn) throws HiveMetaException {
    System.out.println("Validating metastore schema tables");
    String version = null;
    try {
        MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
        version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
    } catch (HiveMetaException he) {
        System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
        System.out.println("Failed in schema table validation.");
        LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage(), he);
        return false;
    }
    Connection hmsConn = schemaTool.getConnectionToMetastore(false);
    LOG.debug("Validating tables in the schema for version " + version);
    List<String> dbTables = new ArrayList<>();
    ResultSet rs = null;
    try {
        String schema = null;
        try {
            schema = hmsConn.getSchema();
        } catch (SQLFeatureNotSupportedException e) {
            LOG.debug("schema is not supported");
        }
        DatabaseMetaData metadata = conn.getMetaData();
        rs = metadata.getTables(null, schema, "%", new String[] { "TABLE" });
        while (rs.next()) {
            String table = rs.getString("TABLE_NAME");
            dbTables.add(table.toLowerCase());
            LOG.debug("Found table " + table + " in HMS dbstore");
        }
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," + e.getMessage(), e);
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                throw new HiveMetaException("Failed to close resultset", e);
            }
        }
    }
    // parse the schema file to determine the tables that are expected to exist
    // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
    List<String> schemaTables = new ArrayList<>();
    List<String> subScripts = new ArrayList<>();
    String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent();
    String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(), schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath();
    try {
        LOG.debug("Parsing schema script " + schemaFile);
        subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        while (subScripts.size() > 0) {
            schemaFile = baseDir + "/" + schemaTool.getDbType() + "/" + subScripts.remove(0);
            LOG.debug("Parsing subscript " + schemaFile);
            subScripts.addAll(findCreateTable(schemaFile, schemaTables));
        }
    } catch (Exception e) {
        System.err.println("Exception in parsing schema file. Cause:" + e.getMessage());
        System.out.println("Failed in schema table validation.");
        return false;
    }
    LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
    LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
    // now diff the lists
    schemaTables.removeAll(dbTables);
    if (schemaTables.size() > 0) {
        Collections.sort(schemaTables);
        System.err.println("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] " + "are missing from the metastore database schema.");
        System.out.println("[FAIL]\n");
        return false;
    } else {
        System.out.println("[SUCCESS]\n");
        return true;
    }
}

Example 55 with HiveMetaException

use of org.apache.hadoop.hive.metastore.HiveMetaException in project hive by apache.

the class SchemaToolTaskValidate method validateSequences.

@VisibleForTesting
boolean validateSequences(Connection conn) throws HiveMetaException {
    Map<String, Pair<String, String>> seqNameToTable = new ImmutableMap.Builder<String, Pair<String, String>>().put("MDatabase", Pair.of("DBS", "DB_ID")).put("MRole", Pair.of("ROLES", "ROLE_ID")).put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID")).put("MTable", Pair.of("TBLS", "TBL_ID")).put("MStorageDescriptor", Pair.of("SDS", "SD_ID")).put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID")).put("MColumnDescriptor", Pair.of("CDS", "CD_ID")).put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID")).put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID")).put("MPartition", Pair.of("PARTITIONS", "PART_ID")).put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID")).put("MFunction", Pair.of("FUNCS", "FUNC_ID")).put("MIndex", Pair.of("IDXS", "INDEX_ID")).put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID")).build();
    System.out.println("Validating sequence number for SEQUENCE_TABLE");
    boolean isValid = true;
    try {
        Statement stmt = conn.createStatement();
        for (Map.Entry<String, Pair<String, String>> e : seqNameToTable.entrySet()) {
            String tableName = e.getValue().getLeft();
            String tableKey = e.getValue().getRight();
            String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + e.getKey();
            String seqQuery = schemaTool.quote(QUERY_SEQ);
            String maxIdQuery = String.format(schemaTool.quote(QUERY_MAX_ID), tableKey, tableName);
            ResultSet res = stmt.executeQuery(maxIdQuery);
            if (res.next()) {
                long maxId = res.getLong(1);
                if (maxId > 0) {
                    PreparedStatement stmtSeq = conn.prepareStatement(seqQuery);
                    stmtSeq.setString(1, fullSequenceName);
                    ResultSet resSeq = stmtSeq.executeQuery();
                    if (!resSeq.next()) {
                        isValid = false;
                        System.err.println("Missing SEQUENCE_NAME " + e.getKey() + " from SEQUENCE_TABLE");
                    } else if (resSeq.getLong(1) < maxId) {
                        isValid = false;
                        System.err.println("NEXT_VAL for " + e.getKey() + " in SEQUENCE_TABLE < max(" + tableKey + ") in " + tableName);
                    }
                }
            }
        }
        System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n");
        return isValid;
    } catch (SQLException e) {
        throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
    }
}