Examples with InvalidOperationException org.apache.hadoop.hive.metastore.api.InvalidOperationException used on opensource projects

Example 11 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class AlterTableTouchOperation method execute.

@Override
public int execute() throws HiveException {
    // TODO: catalog
    Table table = context.getDb().getTable(desc.getTableName());
    EnvironmentContext environmentContext = new EnvironmentContext();
    environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
    if (desc.getPartitionSpec() == null) {
        context.getDb().alterTable(table, false, environmentContext, true);
        context.getWork().getInputs().add(new ReadEntity(table));
        DDLUtils.addIfAbsentByName(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK), context);
    } else {
        Partition part = context.getDb().getPartition(table, desc.getPartitionSpec(), false);
        if (part == null) {
            throw new HiveException("Specified partition does not exist");
        }
        try {
            context.getDb().alterPartition(table.getCatalogName(), table.getDbName(), table.getTableName(), part, environmentContext, true);
        } catch (InvalidOperationException e) {
            throw new HiveException(e);
        }
        context.getWork().getInputs().add(new ReadEntity(part));
        DDLUtils.addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK), context);
    }
    return 0;
}

Example 12 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class Hive method loadPartitionInternal.

/**
 * Move all the files from loadPath into Hive. If the partition
 * does not exist - one is created - files in loadPath are moved into Hive. But the
 * directory itself is not removed.
 *
 * @param loadPath
 *          Directory containing files to load into Table
 * @param tbl
 *          name of table to be loaded.
 * @param partSpec
 *          defines which partition needs to be loaded
 * @param oldPart
 *          already existing partition object, can be null
 * @param loadFileType
 *          if REPLACE_ALL - replace files in the table,
 *          otherwise add files to table (KEEP_EXISTING, OVERWRITE_EXISTING)
 * @param inheritTableSpecs if true, on [re]creating the partition, take the
 *          location/inputformat/outputformat/serde details from table spec
 * @param inheritLocation
 *          if true, partition path is generated from table
 * @param isSkewedStoreAsSubdir
 *          if true, skewed is stored as sub-directory
 * @param isSrcLocal
 *          If the source directory is LOCAL
 * @param isAcidIUDoperation
 *          true if this is an ACID operation Insert/Update/Delete operation
 * @param resetStatistics
 *          if true, reset the statistics. Do not reset statistics if false.
 * @param writeId
 *          write ID allocated for the current load operation
 * @param stmtId
 *          statement ID of the current load statement
 * @param isInsertOverwrite
 * @param isTxnTable
 *
 * @return Partition object being loaded with data
 * @throws HiveException
 */
private Partition loadPartitionInternal(Path loadPath, Table tbl, Map<String, String> partSpec, Partition oldPart, LoadFileType loadFileType, boolean inheritTableSpecs, boolean inheritLocation, boolean isSkewedStoreAsSubdir, boolean isSrcLocal, boolean isAcidIUDoperation, boolean resetStatistics, Long writeId, int stmtId, boolean isInsertOverwrite, boolean isTxnTable, List<FileStatus> newFiles, boolean isDirectInsert) throws HiveException {
    Path tblDataLocationPath = tbl.getDataLocation();
    boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters());
    assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName();
    boolean isFullAcidTable = AcidUtils.isFullAcidTable(tbl);
    List<FileStatus> newFileStatuses = null;
    try {
        PerfLogger perfLogger = SessionState.getPerfLogger();
        /**
         * Move files before creating the partition since down stream processes
         * check for existence of partition in metadata before accessing the data.
         * If partition is created before data is moved, downstream waiting
         * processes might move forward with partial data
         */
        Path oldPartPath = (oldPart != null) ? oldPart.getDataLocation() : null;
        Path newPartPath = null;
        if (inheritLocation) {
            newPartPath = genPartPathFromTable(tbl, partSpec, tblDataLocationPath);
            if (oldPart != null) {
                /*
           * If we are moving the partition across filesystem boundaries
           * inherit from the table properties. Otherwise (same filesystem) use the
           * original partition location.
           *
           * See: HIVE-1707 and HIVE-2117 for background
           */
                FileSystem oldPartPathFS = oldPartPath.getFileSystem(getConf());
                FileSystem loadPathFS = loadPath.getFileSystem(getConf());
                if (FileUtils.equalsFileSystem(oldPartPathFS, loadPathFS)) {
                    newPartPath = oldPartPath;
                }
            }
        } else {
            newPartPath = oldPartPath == null ? genPartPathFromTable(tbl, partSpec, tblDataLocationPath) : oldPartPath;
        }
        perfLogger.perfLogBegin("MoveTask", PerfLogger.FILE_MOVES);
        // Note: this assumes both paths are qualified; which they are, currently.
        if (((isMmTableWrite || isDirectInsert || isFullAcidTable) && loadPath.equals(newPartPath)) || (loadFileType == LoadFileType.IGNORE)) {
            // MM insert query or direct insert; move itself is a no-op.
            if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
                Utilities.FILE_OP_LOGGER.trace("not moving " + loadPath + " to " + newPartPath + " (MM = " + isMmTableWrite + ", Direct insert = " + isDirectInsert + ")");
            }
            if (newFiles != null) {
                if (!newFiles.isEmpty()) {
                    newFileStatuses = new ArrayList<>();
                    newFileStatuses.addAll(newFiles);
                } else {
                    newFileStatuses = listFilesCreatedByQuery(loadPath, writeId, stmtId);
                    newFiles.addAll(newFileStatuses);
                }
            }
            if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
                Utilities.FILE_OP_LOGGER.trace("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
            }
        } else {
            // Either a non-MM query, or a load into MM table from an external source.
            Path destPath = newPartPath;
            if (isMmTableWrite) {
                // We will load into MM directory, and hide previous directories if needed.
                destPath = new Path(destPath, isInsertOverwrite ? AcidUtils.baseDir(writeId) : AcidUtils.deltaSubdir(writeId, writeId, stmtId));
            }
            if (!isAcidIUDoperation && isFullAcidTable) {
                destPath = fixFullAcidPathForLoadData(loadFileType, destPath, writeId, stmtId, tbl);
            }
            if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
                Utilities.FILE_OP_LOGGER.trace("moving " + loadPath + " to " + destPath);
            }
            boolean isManaged = tbl.getTableType() == TableType.MANAGED_TABLE;
            // TODO: why is "&& !isAcidIUDoperation" needed here?
            if (!isTxnTable && ((loadFileType == LoadFileType.REPLACE_ALL) || (oldPart == null && !isAcidIUDoperation))) {
                // for fullAcid tables we don't delete files for commands with OVERWRITE - we create a new
                // base_x.  (there is Insert Overwrite and Load Data Overwrite)
                boolean isSkipTrash = MetaStoreUtils.isSkipTrash(tbl.getParameters());
                boolean needRecycle = !tbl.isTemporary() && ReplChangeManager.shouldEnableCm(getDatabase(tbl.getDbName()), tbl.getTTable());
                replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(), isSrcLocal, isSkipTrash, newFiles, FileUtils.HIDDEN_FILES_PATH_FILTER, needRecycle, isManaged, isInsertOverwrite);
            } else {
                FileSystem fs = destPath.getFileSystem(conf);
                copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation, (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles, tbl.getNumBuckets() > 0, isFullAcidTable, isManaged, false);
            }
        }
        perfLogger.perfLogEnd("MoveTask", PerfLogger.FILE_MOVES);
        Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
        alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
        validatePartition(newTPart);
        // When inserting into a new partition, the add partition event takes care of insert event
        if ((null != oldPart) && (null != newFiles)) {
            if (isTxnTable) {
                addWriteNotificationLog(tbl, partSpec, newFiles, writeId, null);
            } else {
                fireInsertEvent(tbl, partSpec, (loadFileType == LoadFileType.REPLACE_ALL), newFiles);
            }
        } else {
            LOG.debug("No new files were created, and is not a replace, or we're inserting into a " + "partition that does not exist yet. Skipping generating INSERT event.");
        }
        // recreate the partition if it existed before
        if (isSkewedStoreAsSubdir) {
            org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
            SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
            /* Construct list bucketing location mappings from sub-directory name. */
            Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(newPartPath, skewedInfo);
            /* Add list bucketing location mappings. */
            skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
            newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
        }
        // auto gather stats is turn off explicitly. We need to reset the stats in both cases.
        if (resetStatistics || !this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
            LOG.debug("Clear partition column statistics by setting basic stats to false for " + newTPart.getCompleteName());
            StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE);
        }
        if (oldPart == null) {
            newTPart.getTPartition().setParameters(new HashMap<String, String>());
            if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
                StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE);
            }
            // Note: we are creating a brand new the partition, so this is going to be valid for ACID.
            List<FileStatus> filesForStats = null;
            if (newFileStatuses != null && !newFileStatuses.isEmpty()) {
                filesForStats = newFileStatuses;
            } else {
                if (isTxnTable) {
                    filesForStats = AcidUtils.getAcidFilesForStats(newTPart.getTable(), newPartPath, conf, null);
                } else {
                    filesForStats = HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf));
                }
            }
            if (filesForStats != null) {
                MetaStoreServerUtils.populateQuickStats(filesForStats, newTPart.getParameters());
            } else {
                // The ACID state is probably absent. Warning is logged in the get method.
                MetaStoreServerUtils.clearQuickStats(newTPart.getParameters());
            }
        }
        return newTPart;
    } catch (IOException | MetaException | InvalidOperationException e) {
        LOG.error("Error in loadPartitionInternal", e);
        throw new HiveException(e);
    }
}

Example 13 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class PartitionTree method alterPartition.

/**
 * Alter an existing partition. The flow is following:
 * <p>
 *   1) search for existing partition
 *   2) if found delete it
 *   3) insert new partition
 * </p>
 * @param oldPartitionVals the values of existing partition, which is altered, must be not null.
 * @param newPartition the new partition, must be not null.
 * @param isRename true, if rename is requested, meaning that all properties of partition can be changed, except
 *                 of its location.
 * @throws MetaException table or db name is altered.
 * @throws InvalidOperationException the new partition values are null, or the old partition cannot be found.
 */
void alterPartition(List<String> oldPartitionVals, Partition newPartition, boolean isRename) throws MetaException, InvalidOperationException, NoSuchObjectException {
    if (oldPartitionVals == null || oldPartitionVals.isEmpty()) {
        throw new InvalidOperationException("Old partition values cannot be null or empty.");
    }
    if (newPartition == null) {
        throw new InvalidOperationException("New partition cannot be null.");
    }
    Partition oldPartition = getPartition(oldPartitionVals);
    if (oldPartition == null) {
        throw new InvalidOperationException("Partition with partition values " + Arrays.toString(oldPartitionVals.toArray()) + " is not found.");
    }
    if (!oldPartition.getDbName().equals(newPartition.getDbName())) {
        throw new MetaException("Db name cannot be altered.");
    }
    if (!oldPartition.getTableName().equals(newPartition.getTableName())) {
        throw new MetaException("Table name cannot be altered.");
    }
    if (isRename) {
        newPartition.getSd().setLocation(oldPartition.getSd().getLocation());
    }
    dropPartition(oldPartitionVals);
    String partName = makePartName(tTable.getPartitionKeys(), newPartition.getValues());
    if (parts.containsKey(partName)) {
        throw new InvalidOperationException("Partition " + partName + " already exists");
    }
    parts.put(partName, newPartition);
}

Example 14 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class PartitionTree method alterPartitions.

/**
 * Alter multiple partitions. This operation is transactional.
 * @param newParts list of new partitions, must be not null.
 * @throws MetaException table or db name is altered.
 * @throws InvalidOperationException the new partition values are null, or the old partition cannot be found.
 * @throws NoSuchObjectException the old partition cannot be found.
 */
void alterPartitions(List<Partition> newParts) throws MetaException, InvalidOperationException, NoSuchObjectException {
    // altering partitions in a batch must be transactional, therefore bofore starting the altering, clone the original
    // partitions map. If something fails, revert it back.
    Map<String, Partition> clonedPartitions = new LinkedHashMap<>();
    parts.forEach((key, value) -> clonedPartitions.put(key, new Partition(value)));
    for (Partition partition : newParts) {
        try {
            if (partition == null) {
                throw new InvalidOperationException("New partition cannot be null.");
            }
            alterPartition(partition.getValues(), partition, false);
        } catch (MetaException | InvalidOperationException | NoSuchObjectException e) {
            parts = clonedPartitions;
            throw e;
        }
    }
}

Example 15 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class Hive method renamePartition.

/**
 * Rename a old partition to new partition
 *
 * @param tbl
 *          existing table
 * @param oldPartSpec
 *          spec of old partition
 * @param newPart
 *          new partition
 * @throws HiveException
 */
public void renamePartition(Table tbl, Map<String, String> oldPartSpec, Partition newPart, long replWriteId) throws HiveException {
    try {
        Map<String, String> newPartSpec = newPart.getSpec();
        if (oldPartSpec.keySet().size() != tbl.getPartCols().size() || newPartSpec.keySet().size() != tbl.getPartCols().size()) {
            throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. ");
        }
        if (!oldPartSpec.keySet().equals(newPartSpec.keySet())) {
            throw new HiveException("Unable to rename partition to the same name: old and new partition cols don't match. ");
        }
        List<String> pvals = new ArrayList<String>();
        for (FieldSchema field : tbl.getPartCols()) {
            String val = oldPartSpec.get(field.getName());
            if (val == null || val.length() == 0) {
                throw new HiveException("get partition: Value for key " + field.getName() + " is null or empty");
            } else if (val != null) {
                pvals.add(val);
            }
        }
        String validWriteIds = null;
        if (AcidUtils.isTransactionalTable(tbl)) {
            TableSnapshot tableSnapshot;
            if (replWriteId > 0) {
                // We need a valid writeId list for a transactional table modification. During
                // replication we do not have a valid writeId list which was used to modify the table
                // on the source. But we know for sure that the writeId associated with it was valid
                // then (otherwise modification would have failed on the source). So use a valid
                // transaction list with only that writeId.
                ValidWriteIdList writeIds = new ValidReaderWriteIdList(TableName.getDbTable(tbl.getDbName(), tbl.getTableName()), new long[0], new BitSet(), replWriteId);
                tableSnapshot = new TableSnapshot(replWriteId, writeIds.writeToString());
            } else {
                // Set table snapshot to api.Table to make it persistent.
                tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, true);
            }
            if (tableSnapshot != null) {
                newPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
                validWriteIds = tableSnapshot.getValidWriteIdList();
            }
        }
        String catName = (tbl.getCatalogName() != null) ? tbl.getCatalogName() : getDefaultCatalog(conf);
        getMSC().renamePartition(catName, tbl.getDbName(), tbl.getTableName(), pvals, newPart.getTPartition(), validWriteIds);
    } catch (InvalidOperationException e) {
        throw new HiveException("Unable to rename partition. " + e.getMessage(), e);
    } catch (MetaException e) {
        throw new HiveException("Unable to rename partition. " + e.getMessage(), e);
    } catch (TException e) {
        throw new HiveException("Unable to rename partition. " + e.getMessage(), e);
    }
}