Examples with Warehouse org.apache.hadoop.hive.metastore.Warehouse used on opensource projects

Example 6 with Warehouse

use of org.apache.hadoop.hive.metastore.Warehouse in project hive by apache.

the class AlterTableArchiveUtils method deleteDir.

static void deleteDir(Path dir, boolean shouldEnableCm, Configuration conf) throws HiveException {
    try {
        Warehouse wh = new Warehouse(conf);
        wh.deleteDir(dir, true, false, shouldEnableCm);
    } catch (MetaException e) {
        throw new HiveException(e);
    }
}

Example 7 with Warehouse

use of org.apache.hadoop.hive.metastore.Warehouse in project hive by apache.

the class TestHive method validateTable.

/**
 * Gets a table from the metastore and compares it to the original Table
 *
 * @param tbl
 * @param tableName
 * @throws MetaException
 */
private void validateTable(Table tbl, String tableName) throws MetaException {
    Warehouse wh = new Warehouse(hiveConf);
    Table ft = null;
    try {
        // hm.getTable result will not have privileges set (it does not retrieve
        // that part from metastore), so unset privileges to null before comparing
        // (create table sets it to empty (non null) structures)
        tbl.getTTable().setPrivilegesIsSet(false);
        ft = hm.getTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
        Assert.assertTrue(ft.getTTable().isSetId());
        ft.getTTable().unsetId();
        assertNotNull("Unable to fetch table", ft);
        ft.checkValidity(hiveConf);
        assertEquals("Table names didn't match for table: " + tableName, tbl.getTableName(), ft.getTableName());
        assertEquals("Table owners didn't match for table: " + tableName, tbl.getOwner(), ft.getOwner());
        assertEquals("Table retention didn't match for table: " + tableName, tbl.getRetention(), ft.getRetention());
        assertEquals("Data location is not set correctly", wh.getDefaultTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(), ft.getDataLocation().toString());
        // now that URI and times are set correctly, set the original table's uri and times
        // and then compare the two tables
        tbl.setDataLocation(ft.getDataLocation());
        tbl.setCreateTime(ft.getTTable().getCreateTime());
        tbl.getParameters().put(hive_metastoreConstants.DDL_TIME, ft.getParameters().get(hive_metastoreConstants.DDL_TIME));
        // Txn stuff set by metastore
        if (tbl.getTTable().isSetWriteId() != ft.getTTable().isSetWriteId()) {
            // No need to compare this field.
            ft.getTTable().setWriteId(0);
            tbl.getTTable().setWriteId(0);
        }
        // accessType set by HMS Transformer
        if (tbl.getTTable().isSetAccessType() != ft.getTTable().isSetAccessType()) {
            // No need to compare this field.
            tbl.getTTable().setAccessType(ft.getTTable().getAccessType());
        }
        tbl.getTTable().unsetId();
        assertTrue("Tables  doesn't match: " + tableName + " (" + ft.getTTable() + "; " + tbl.getTTable() + ")", ft.getTTable().equals(tbl.getTTable()));
        assertEquals("SerializationLib is not set correctly", tbl.getSerializationLib(), ft.getSerializationLib());
        assertEquals("Serde is not set correctly", tbl.getDeserializer().getClass().getName(), ft.getDeserializer().getClass().getName());
    } catch (HiveException e) {
        System.err.println(StringUtils.stringifyException(e));
        assertTrue("Unable to fetch table correctly: " + tableName, false);
    }
}

Example 8 with Warehouse

use of org.apache.hadoop.hive.metastore.Warehouse in project hive by apache.

the class SessionHiveMetaStoreClient method createTempTable.

private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, EnvironmentContext envContext) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
    boolean isVirtualTable = tbl.getTableName().startsWith(SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX);
    SessionState ss = SessionState.get();
    if (ss == null) {
        throw new MetaException("No current SessionState, cannot create temporary table: " + Warehouse.getQualifiedName(tbl));
    }
    // We may not own the table object, create a copy
    tbl = deepCopyAndLowerCaseTable(tbl);
    String dbName = tbl.getDbName();
    String tblName = tbl.getTableName();
    Map<String, Table> tables = getTempTablesForDatabase(dbName, tblName);
    if (tables != null && tables.containsKey(tblName)) {
        throw new MetaException("Temporary table " + StatsUtils.getFullyQualifiedTableName(dbName, tblName) + " already exists");
    }
    // Create temp table directory
    Warehouse wh = getWh();
    if (tbl.getSd().getLocation() == null) {
        // Temp tables that do not go through SemanticAnalyzer may not have location set - do it here.
        // For example export of acid tables generates a query plan that creates a temp table.
        tbl.getSd().setLocation(SessionState.generateTempTableLocation(conf));
    }
    Path tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation()));
    if (tblPath == null) {
        throw new MetaException("Temp table path not set for " + tbl.getTableName());
    } else {
        if (!wh.isDir(tblPath)) {
            if (!wh.mkdirs(tblPath)) {
                throw new MetaException(tblPath + " is not a directory or unable to create one");
            }
        }
        // Make sure location string is in proper format
        tbl.getSd().setLocation(tblPath.toString());
    }
    // Add temp table info to current session
    Table tTable = new Table(tbl);
    if (!isVirtualTable) {
        StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), getColumnNamesForTable(tbl), StatsSetupConst.TRUE);
    }
    if (tables == null) {
        tables = new HashMap<String, Table>();
        ss.getTempTables().put(dbName, tables);
    }
    tables.put(tblName, tTable);
    createPartitionedTempTable(tbl);
}

Example 9 with Warehouse

use of org.apache.hadoop.hive.metastore.Warehouse in project hive by apache.

the class ImportSemanticAnalyzer method prepareImport.

public static boolean prepareImport(boolean isImportCmd, boolean isLocationSet, boolean isExternalSet, boolean isPartSpecSet, boolean waitOnPrecursor, String parsedLocation, String parsedTableName, String overrideDBName, LinkedHashMap<String, String> parsedPartSpec, String fromLocn, EximUtil.SemanticAnalyzerWrapperContext x, UpdatedMetaDataTracker updatedMetadata, HiveTxnManager txnMgr, // Initialize with 0 for non-ACID and non-MM tables.
long writeId, MetaData rv, String dumpRoot, ReplicationMetricCollector metricCollector) throws IOException, MetaException, HiveException, URISyntaxException {
    // initialize load path
    URI fromURI = EximUtil.getValidatedURI(x.getConf(), stripQuotes(fromLocn));
    Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath());
    FileSystem fs = FileSystem.get(fromURI, x.getConf());
    x.getInputs().add(toReadEntity(fromPath, x.getConf()));
    if (rv.getTable() == null) {
        // nothing to do here, silently return.
        return false;
    }
    ReplicationSpec replicationSpec = rv.getReplicationSpec();
    if (replicationSpec.isNoop()) {
        // nothing to do here, silently return.
        x.getLOG().debug("Current update with ID:{} is noop", replicationSpec.getCurrentReplicationState());
        return false;
    }
    if (isImportCmd) {
        replicationSpec.setReplSpecType(ReplicationSpec.Type.IMPORT);
    }
    String dbname = rv.getTable().getDbName();
    if ((overrideDBName != null) && (!overrideDBName.isEmpty())) {
        // If the parsed statement contained a db.tablename specification, prefer that.
        dbname = overrideDBName;
    }
    // Create table associated with the import
    // Executed if relevant, and used to contain all the other details about the table if not.
    ImportTableDesc tblDesc;
    org.apache.hadoop.hive.metastore.api.Table tblObj = rv.getTable();
    try {
        tblDesc = getBaseCreateTableDescFromTable(dbname, tblObj);
    } catch (Exception e) {
        throw new HiveException(e);
    }
    boolean inReplicationScope = false;
    if ((replicationSpec != null) && replicationSpec.isInReplicationScope()) {
        tblDesc.setReplicationSpec(replicationSpec);
        inReplicationScope = true;
        tblDesc.setReplWriteId(writeId);
        tblDesc.setOwnerName(tblObj.getOwner());
    }
    if (isExternalSet) {
        tblDesc.setExternal(isExternalSet);
    // This condition-check could have been avoided, but to honour the old
    // default of not calling if it wasn't set, we retain that behaviour.
    // TODO:cleanup after verification that the outer if isn't really needed here
    }
    if (isLocationSet) {
        STATIC_LOG.debug("table {} location is {}", tblDesc.getTableName(), parsedLocation);
        tblDesc.setLocation(parsedLocation);
        x.getInputs().add(toReadEntity(new Path(parsedLocation), x.getConf()));
    }
    if (StringUtils.isNotBlank(parsedTableName)) {
        tblDesc.setTableName(TableName.fromString(parsedTableName, null, dbname));
    }
    if (tblDesc.getTableName() == null) {
        // Either we got the tablename from the IMPORT statement (first priority) or from the export dump.
        throw new SemanticException(ErrorMsg.NEED_TABLE_SPECIFICATION.getMsg());
    } else {
        x.getConf().set("import.destination.table", tblDesc.getTableName());
    }
    List<AlterTableAddPartitionDesc> partitionDescs = new ArrayList<>();
    Iterable<Partition> partitions = rv.getPartitions();
    for (Partition partition : partitions) {
        // TODO: this should ideally not create AddPartitionDesc per partition
        AlterTableAddPartitionDesc partsDesc = getBaseAddPartitionDescFromPartition(fromPath, dbname, tblDesc, partition, replicationSpec, x.getConf());
        partitionDescs.add(partsDesc);
    }
    if (isPartSpecSet) {
        // The import specification asked for only a particular partition to be loaded
        // We load only that, and ignore all the others.
        boolean found = false;
        for (Iterator<AlterTableAddPartitionDesc> partnIter = partitionDescs.listIterator(); partnIter.hasNext(); ) {
            AlterTableAddPartitionDesc addPartitionDesc = partnIter.next();
            if (!found && addPartitionDesc.getPartitions().get(0).getPartSpec().equals(parsedPartSpec)) {
                found = true;
            } else {
                partnIter.remove();
            }
        }
        if (!found) {
            throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(" - Specified partition not found in import directory"));
        }
    }
    Warehouse wh = new Warehouse(x.getConf());
    Table table = tableIfExists(tblDesc, x.getHive());
    boolean tableExists = false;
    if (table != null) {
        checkTable(table, tblDesc, replicationSpec, x.getConf());
        x.getLOG().debug("table " + tblDesc.getTableName() + " exists: metadata checked");
        tableExists = true;
    }
    if (!tableExists && isExternalSet) {
        // If the user is explicitly importing a new external table, clear txn flags from the spec.
        AcidUtils.setNonTransactional(tblDesc.getTblProps());
    }
    int stmtId = 0;
    if (!replicationSpec.isInReplicationScope() && ((tableExists && AcidUtils.isTransactionalTable(table)) || (!tableExists && AcidUtils.isTablePropertyTransactional(tblDesc.getTblProps())))) {
        // In replication flow, no need to allocate write id. It will be allocated using the alloc write id event.
        if (x.getCtx().getExplainConfig() == null && !inReplicationScope) {
            writeId = txnMgr.getTableWriteId(tblDesc.getDatabaseName(), tblDesc.getTableName());
            stmtId = txnMgr.getStmtIdAndIncrement();
        }
    }
    if (inReplicationScope) {
        createReplImportTasks(tblDesc, partitionDescs, replicationSpec, waitOnPrecursor, table, fromURI, wh, x, writeId, stmtId, updatedMetadata, dumpRoot, metricCollector);
    } else {
        createRegularImportTasks(tblDesc, partitionDescs, isPartSpecSet, replicationSpec, table, fromURI, fs, wh, x, writeId, stmtId);
    }
    return tableExists;
}

Example 10 with Warehouse

use of org.apache.hadoop.hive.metastore.Warehouse in project hive by apache.

the class MetaDataExportListener method export_meta_data.

/**
 * Export the metadata to a given path, and then move it to the user's trash
 */
private void export_meta_data(PreDropTableEvent tableEvent) throws MetaException {
    FileSystem fs = null;
    Table tbl = tableEvent.getTable();
    String name = tbl.getTableName();
    org.apache.hadoop.hive.ql.metadata.Table mTbl = new org.apache.hadoop.hive.ql.metadata.Table(tbl);
    IHMSHandler handler = tableEvent.getHandler();
    Configuration conf = handler.getConf();
    Warehouse wh = new Warehouse(conf);
    Path tblPath = new Path(tbl.getSd().getLocation());
    fs = wh.getFs(tblPath);
    Date now = new Date();
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
    String dateString = sdf.format(now);
    String exportPathString = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METADATA_EXPORT_LOCATION);
    boolean moveMetadataToTrash = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.MOVE_EXPORTED_METADATA_TO_TRASH);
    Path exportPath = null;
    if (exportPathString != null && exportPathString.length() == 0) {
        exportPath = fs.getHomeDirectory();
    } else {
        exportPath = new Path(exportPathString);
    }
    Path metaPath = new Path(exportPath, name + "." + dateString);
    LOG.info("Exporting the metadata of table " + tbl.toString() + " to path " + metaPath.toString());
    try {
        fs.mkdirs(metaPath);
    } catch (IOException e) {
        throw new MetaException(e.getMessage());
    }
    Path outFile = new Path(metaPath, name + EximUtil.METADATA_NAME);
    try {
        SessionState.getConsole().printInfo("Beginning metadata export");
        EximUtil.createExportDump(fs, outFile, mTbl, null, null, new HiveConf(conf, MetaDataExportListener.class));
        if (moveMetadataToTrash == true) {
            wh.deleteDir(metaPath, true, false, false);
        }
    } catch (IOException | SemanticException e) {
        throw new MetaException(e.getMessage());
    }
}