Examples with MStorageDescriptor org.apache.hadoop.hive.metastore.model.MStorageDescriptor used on opensource projects

Example 6 with MStorageDescriptor

use of org.apache.hadoop.hive.metastore.model.MStorageDescriptor in project hive by apache.

the class ObjectStore method listStorageDescriptorsWithCD.

/**
   * Get a list of storage descriptors that reference a particular Column Descriptor
   * @param oldCD the column descriptor to get storage descriptors for
   * @param maxSDs the maximum number of SDs to return
   * @return a list of storage descriptors
   */
private List<MStorageDescriptor> listStorageDescriptorsWithCD(MColumnDescriptor oldCD, long maxSDs, QueryWrapper queryWrapper) {
    boolean success = false;
    List<MStorageDescriptor> sds = null;
    try {
        openTransaction();
        LOG.debug("Executing listStorageDescriptorsWithCD");
        Query query = queryWrapper.query = pm.newQuery(MStorageDescriptor.class, "this.cd == inCD");
        query.declareParameters("MColumnDescriptor inCD");
        if (maxSDs >= 0) {
            // User specified a row limit, set it on the Query
            query.setRange(0, maxSDs);
        }
        sds = (List<MStorageDescriptor>) query.execute(oldCD);
        LOG.debug("Done executing query for listStorageDescriptorsWithCD");
        pm.retrieveAll(sds);
        success = commitTransaction();
        LOG.debug("Done retrieving all objects for listStorageDescriptorsWithCD");
    } finally {
        if (!success) {
            rollbackTransaction();
        }
    }
    return sds;
}

Example 7 with MStorageDescriptor

use of org.apache.hadoop.hive.metastore.model.MStorageDescriptor in project hive by apache.

the class ObjectStore method alterPartitionNoTxn.

/**
 * Alters an existing partition. Initiates copy of SD. Returns the old CD.
 * @param dbname
 * @param name
 * @param part_vals Partition values (of the original partition instance)
 * @param newPart Partition object containing new information
 * @return The column descriptor of the old partition instance (null if table is a view)
 * @throws InvalidObjectException
 * @throws MetaException
 */
private MColumnDescriptor alterPartitionNoTxn(String dbname, String name, List<String> part_vals, Partition newPart) throws InvalidObjectException, MetaException {
    name = normalizeIdentifier(name);
    dbname = normalizeIdentifier(dbname);
    MPartition oldp = getMPartition(dbname, name, part_vals);
    MPartition newp = convertToMPart(newPart, false);
    MColumnDescriptor oldCD = null;
    MStorageDescriptor oldSD = oldp.getSd();
    if (oldSD != null) {
        oldCD = oldSD.getCD();
    }
    if (oldp == null || newp == null) {
        throw new InvalidObjectException("partition does not exist.");
    }
    oldp.setValues(newp.getValues());
    oldp.setPartitionName(newp.getPartitionName());
    oldp.setParameters(newPart.getParameters());
    if (!TableType.VIRTUAL_VIEW.name().equals(oldp.getTable().getTableType())) {
        copyMSD(newp.getSd(), oldp.getSd());
    }
    if (newp.getCreateTime() != oldp.getCreateTime()) {
        oldp.setCreateTime(newp.getCreateTime());
    }
    if (newp.getLastAccessTime() != oldp.getLastAccessTime()) {
        oldp.setLastAccessTime(newp.getLastAccessTime());
    }
    return oldCD;
}

Example 8 with MStorageDescriptor

use of org.apache.hadoop.hive.metastore.model.MStorageDescriptor in project hive by apache.

the class ObjectStore method alterTable.

@Override
public Table alterTable(String catName, String dbname, String name, Table newTable, String queryValidWriteIds) throws InvalidObjectException, MetaException {
    boolean success = false;
    try {
        openTransaction();
        name = normalizeIdentifier(name);
        dbname = normalizeIdentifier(dbname);
        catName = normalizeIdentifier(catName);
        MTable newt = convertToMTable(newTable);
        if (newt == null) {
            throw new InvalidObjectException("new table is invalid");
        }
        MTable oldt = getMTable(catName, dbname, name);
        if (oldt == null) {
            throw new MetaException("table " + dbname + "." + name + " doesn't exist");
        }
        // For now only alter name, owner, parameters, cols, bucketcols are allowed
        oldt.setDatabase(newt.getDatabase());
        oldt.setTableName(normalizeIdentifier(newt.getTableName()));
        boolean isTxn = TxnUtils.isTransactionalTable(newTable);
        boolean isToTxn = isTxn && !TxnUtils.isTransactionalTable(oldt.getParameters());
        if (!isToTxn && isTxn && areTxnStatsSupported) {
            // Transactional table is altered without a txn. Make sure there are no changes to the flag.
            String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(name, dbname), oldt.getParameters(), newTable.getParameters(), newTable.getWriteId(), queryValidWriteIds, false);
            if (errorMsg != null) {
                throw new MetaException(errorMsg);
            }
        }
        oldt.setParameters(newt.getParameters());
        oldt.setOwner(newt.getOwner());
        oldt.setOwnerType(newt.getOwnerType());
        // Fully copy over the contents of the new SD into the old SD,
        // so we don't create an extra SD in the metastore db that has no references.
        MColumnDescriptor oldCD = null;
        MStorageDescriptor oldSD = oldt.getSd();
        if (oldSD != null) {
            oldCD = oldSD.getCD();
        }
        copyMSD(newt.getSd(), oldt.getSd());
        removeUnusedColumnDescriptor(oldCD);
        oldt.setRetention(newt.getRetention());
        oldt.setPartitionKeys(newt.getPartitionKeys());
        oldt.setTableType(newt.getTableType());
        oldt.setLastAccessTime(newt.getLastAccessTime());
        oldt.setViewOriginalText(newt.getViewOriginalText());
        oldt.setViewExpandedText(newt.getViewExpandedText());
        oldt.setRewriteEnabled(newt.isRewriteEnabled());
        // Set stats invalid for ACID conversion; it doesn't pass in the write ID.
        if (isTxn) {
            if (!areTxnStatsSupported || isToTxn) {
                StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE);
            } else if (queryValidWriteIds != null && newTable.getWriteId() > 0) {
                // Check concurrent INSERT case and set false to the flag.
                if (!isCurrentStatsValidForTheQuery(oldt, queryValidWriteIds, true)) {
                    StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE);
                    LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name + ". will be made persistent.");
                }
                assert newTable.getWriteId() > 0;
                oldt.setWriteId(newTable.getWriteId());
            }
        }
        newTable = convertToTable(oldt);
        // commit the changes
        success = commitTransaction();
    } finally {
        if (!success) {
            rollbackTransaction();
        }
    }
    return newTable;
}

Example 9 with MStorageDescriptor

use of org.apache.hadoop.hive.metastore.model.MStorageDescriptor in project hive by apache.

the class ObjectStore method hasRemainingCDReference.

/**
 * Checks if a column descriptor has any remaining references by storage descriptors
 * in the db.
 * @param oldCD the column descriptor to check if it has references or not
 * @return true if has references
 */
private boolean hasRemainingCDReference(MColumnDescriptor oldCD) {
    assert oldCD != null;
    Query query = null;
    /**
     * In order to workaround oracle not supporting limit statement caused performance issue, HIVE-9447 makes
     * all the backend DB run select count(1) from SDS where SDS.CD_ID=? to check if the specific CD_ID is
     * referenced in SDS table before drop a partition. This select count(1) statement does not scale well in
     * Postgres, and there is no index for CD_ID column in SDS table.
     * For a SDS table with with 1.5 million rows, select count(1) has average 700ms without index, while in
     * 10-20ms with index. But the statement before
     * HIVE-9447( SELECT * FROM "SDS" "A0" WHERE "A0"."CD_ID" = $1 limit 1) uses less than 10ms .
     */
    try {
        // HIVE-21075: Fix Postgres performance regression caused by HIVE-9447
        LOG.debug("The dbType is {} ", dbType.getHiveSchemaPostfix());
        if (dbType.isPOSTGRES() || dbType.isMYSQL()) {
            query = pm.newQuery(MStorageDescriptor.class, "this.cd == inCD");
            query.declareParameters("MColumnDescriptor inCD");
            List<MStorageDescriptor> referencedSDs = null;
            LOG.debug("Executing listStorageDescriptorsWithCD");
            // User specified a row limit, set it on the Query
            query.setRange(0L, 1L);
            referencedSDs = (List<MStorageDescriptor>) query.execute(oldCD);
            LOG.debug("Done executing query for listStorageDescriptorsWithCD");
            pm.retrieveAll(referencedSDs);
            LOG.debug("Done retrieving all objects for listStorageDescriptorsWithCD");
            // if no other SD references this CD, we can throw it out.
            return referencedSDs != null && !referencedSDs.isEmpty();
        } else {
            query = pm.newQuery("select count(1) from org.apache.hadoop.hive.metastore.model.MStorageDescriptor where (this.cd == inCD)");
            query.declareParameters("MColumnDescriptor inCD");
            long count = (Long) query.execute(oldCD);
            // if no other SD references this CD, we can throw it out.
            return count != 0;
        }
    } finally {
        if (query != null) {
            query.closeAll();
        }
    }
}

Example 10 with MStorageDescriptor

use of org.apache.hadoop.hive.metastore.model.MStorageDescriptor in project hive by apache.

the class ObjectStore method updateMStorageDescriptorTblPropURI.

/**
 * The following APIs
 *
 *  - updateMStorageDescriptorTblPropURI
 *
 * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
 */
@Deprecated
public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, boolean isDryRun) {
    boolean committed = false;
    Query query = null;
    Map<String, String> updateLocations = new HashMap<>();
    List<String> badRecords = new ArrayList<>();
    UpdatePropURIRetVal retVal = null;
    try {
        openTransaction();
        query = pm.newQuery(MStorageDescriptor.class);
        List<MStorageDescriptor> mSDSs = (List<MStorageDescriptor>) query.execute();
        pm.retrieveAll(mSDSs);
        for (MStorageDescriptor mSDS : mSDSs) {
            updatePropURIHelper(oldLoc, newLoc, tblPropKey, isDryRun, badRecords, updateLocations, mSDS.getParameters());
        }
        committed = commitTransaction();
        if (committed) {
            retVal = new UpdatePropURIRetVal(badRecords, updateLocations);
        }
        return retVal;
    } finally {
        rollbackAndCleanup(committed, query);
    }
}