Examples with InvalidOperationException org.apache.hadoop.hive.metastore.api.InvalidOperationException used on opensource projects

Example 1 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class HiveAlterHandler method alterPartitions.

@Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<Partition> new_parts, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    List<Partition> oldParts = new ArrayList<Partition>();
    List<List<String>> partValsList = new ArrayList<List<String>>();
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partitions because table or database does not exist.");
    }
    boolean success = false;
    try {
        msdb.openTransaction();
        for (Partition tmpPart : new_parts) {
            // Set DDL time to now if not specified
            if (tmpPart.getParameters() == null || tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
                tmpPart.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
            }
            Partition oldTmpPart = msdb.getPartition(dbname, name, tmpPart.getValues());
            oldParts.add(oldTmpPart);
            partValsList.add(tmpPart.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) {
                // Check if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
                    MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
        }
        msdb.alterPartitions(dbname, name, partValsList, new_parts);
        Iterator<Partition> oldPartsIt = oldParts.iterator();
        for (Partition newPart : new_parts) {
            Partition oldPart;
            if (oldPartsIt.hasNext()) {
                oldPart = oldPartsIt.next();
            } else {
                throw new InvalidOperationException("Missing old partition corresponding to new partition " + "when invoking MetaStoreEventListener for alterPartitions event.");
            }
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, newPart, tbl, true, handler);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
        }
        success = msdb.commitTransaction();
    } catch (InvalidObjectException | NoSuchObjectException e) {
        throw new InvalidOperationException("Alter partition operation failed: " + e);
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
    }
    return oldParts;
}

Example 2 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class HiveAlterHandler method alterPartition.

@Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
    boolean success = false;
    Path srcPath = null;
    Path destPath = null;
    FileSystem srcFs = null;
    FileSystem destFs;
    Partition oldPart = null;
    String oldPartLoc = null;
    String newPartLoc = null;
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    // Set DDL time to now if not specified
    if (new_part.getParameters() == null || new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
        new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
    }
    Table tbl = msdb.getTable(dbname, name);
    if (tbl == null) {
        throw new InvalidObjectException("Unable to alter partition because table or database does not exist.");
    }
    //alter partition
    if (part_vals == null || part_vals.size() == 0) {
        try {
            msdb.openTransaction();
            oldPart = msdb.getPartition(dbname, name, new_part.getValues());
            if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                // if stats are same, no need to update
                if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
                    MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
                } else {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
            }
            updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
            msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
            if (transactionalListeners != null && transactionalListeners.size() > 0) {
                AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
                alterPartitionEvent.setEnvironmentContext(environmentContext);
                for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                    transactionalListener.onAlterPartition(alterPartitionEvent);
                }
            }
            success = msdb.commitTransaction();
        } catch (InvalidObjectException e) {
            throw new InvalidOperationException("alter is not possible");
        } catch (NoSuchObjectException e) {
            //old partition does not exist
            throw new InvalidOperationException("alter is not possible");
        } finally {
            if (!success) {
                msdb.rollbackTransaction();
            }
        }
        return oldPart;
    }
    //rename partition
    try {
        msdb.openTransaction();
        try {
            oldPart = msdb.getPartition(dbname, name, part_vals);
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            throw new InvalidObjectException("Unable to rename partition because old partition does not exist");
        }
        Partition check_part;
        try {
            check_part = msdb.getPartition(dbname, name, new_part.getValues());
        } catch (NoSuchObjectException e) {
            // this means there is no existing partition
            check_part = null;
        }
        if (check_part != null) {
            throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + new_part.getValues());
        }
        // if the external partition is renamed, the file should not change
        if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
            new_part.getSd().setLocation(oldPart.getSd().getLocation());
            String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
            try {
                //existing partition column stats is no longer valid, remove
                msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
            } catch (NoSuchObjectException nsoe) {
            //ignore
            } catch (InvalidInputException iie) {
                throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
            }
            msdb.alterPartition(dbname, name, part_vals, new_part);
        } else {
            try {
                destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
                destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
            } catch (NoSuchObjectException e) {
                LOG.debug("Didn't find object in metastore ", e);
                throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
            }
            if (destPath != null) {
                newPartLoc = destPath.toString();
                oldPartLoc = oldPart.getSd().getLocation();
                LOG.info("srcPath:" + oldPartLoc);
                LOG.info("descPath:" + newPartLoc);
                srcPath = new Path(oldPartLoc);
                srcFs = wh.getFs(srcPath);
                destFs = wh.getFs(destPath);
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("New table location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported.");
                }
                try {
                    srcFs.exists(srcPath);
                    if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + tbl.getDbName() + "." + tbl.getTableName() + " already exists : " + destPath);
                    }
                } catch (IOException e) {
                    throw new InvalidOperationException("Unable to access new location " + destPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
                }
                new_part.getSd().setLocation(newPartLoc);
                if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
                    MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
                }
                String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
                try {
                    //existing partition column stats is no longer valid, remove
                    msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
                } catch (NoSuchObjectException nsoe) {
                //ignore
                } catch (InvalidInputException iie) {
                    throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
                }
                msdb.alterPartition(dbname, name, part_vals, new_part);
            }
        }
        if (transactionalListeners != null && transactionalListeners.size() > 0) {
            AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
            alterPartitionEvent.setEnvironmentContext(environmentContext);
            for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                transactionalListener.onAlterPartition(alterPartitionEvent);
            }
        }
        success = msdb.commitTransaction();
    } finally {
        if (!success) {
            msdb.rollbackTransaction();
        }
        if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
            //rename the data directory
            try {
                if (srcFs.exists(srcPath)) {
                    //if destPath's parent path doesn't exist, we should mkdir it
                    Path destParentPath = destPath.getParent();
                    if (!wh.mkdirs(destParentPath, true)) {
                        throw new IOException("Unable to create path " + destParentPath);
                    }
                    wh.renameDir(srcPath, destPath, true);
                    LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
                }
            } catch (IOException ex) {
                LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, ex);
                boolean revertMetaDataTransaction = false;
                try {
                    msdb.openTransaction();
                    msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
                    if (transactionalListeners != null && transactionalListeners.size() > 0) {
                        AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(new_part, oldPart, tbl, true, handler);
                        alterPartitionEvent.setEnvironmentContext(environmentContext);
                        for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                            transactionalListener.onAlterPartition(alterPartitionEvent);
                        }
                    }
                    revertMetaDataTransaction = msdb.commitTransaction();
                } catch (Exception ex2) {
                    LOG.error("Attempt to revert partition metadata change failed. The revert was attempted " + "because associated filesystem rename operation failed with exception " + ex.getMessage(), ex2);
                    if (!revertMetaDataTransaction) {
                        msdb.rollbackTransaction();
                    }
                }
                throw new InvalidOperationException("Unable to access old location " + srcPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + part_vals);
            }
        }
    }
    return oldPart;
}

Example 3 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class HiveAlterHandler method alterTable.

@Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, MetaException {
    final boolean cascade = environmentContext != null && environmentContext.isSetProperties() && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE));
    if (newt == null) {
        throw new InvalidOperationException("New table is invalid: " + newt);
    }
    if (!MetaStoreUtils.validateName(newt.getTableName(), hiveConf)) {
        throw new InvalidOperationException(newt.getTableName() + " is not a valid object name");
    }
    String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
    if (validate != null) {
        throw new InvalidOperationException("Invalid column " + validate);
    }
    Path srcPath = null;
    FileSystem srcFs = null;
    Path destPath = null;
    FileSystem destFs = null;
    boolean success = false;
    boolean dataWasMoved = false;
    boolean rename = false;
    Table oldt = null;
    List<MetaStoreEventListener> transactionalListeners = null;
    if (handler != null) {
        transactionalListeners = handler.getTransactionalListeners();
    }
    try {
        msdb.openTransaction();
        name = name.toLowerCase();
        dbname = dbname.toLowerCase();
        // check if table with the new name already exists
        if (!newt.getTableName().equalsIgnoreCase(name) || !newt.getDbName().equalsIgnoreCase(dbname)) {
            if (msdb.getTable(newt.getDbName(), newt.getTableName()) != null) {
                throw new InvalidOperationException("new table " + newt.getDbName() + "." + newt.getTableName() + " already exists");
            }
            rename = true;
        }
        // get old table
        oldt = msdb.getTable(dbname, name);
        if (oldt == null) {
            throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
        }
        if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, false)) {
            // Throws InvalidOperationException if the new column types are not
            // compatible with the current column types.
            MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(oldt.getSd().getCols(), newt.getSd().getCols());
        }
        if (cascade) {
            //Currently only column related changes can be cascaded in alter table
            if (MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
                List<Partition> parts = msdb.getPartitions(dbname, name, -1);
                for (Partition part : parts) {
                    List<FieldSchema> oldCols = part.getSd().getCols();
                    part.getSd().setCols(newt.getSd().getCols());
                    String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
                    updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
                    msdb.alterPartition(dbname, name, part.getValues(), part);
                }
            } else {
                LOG.warn("Alter table does not cascade changes to its partitions.");
            }
        }
        //check that partition keys have not changed, except for virtual views
        //however, allow the partition comments to change
        boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), newt.getPartitionKeys());
        if (!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
            if (oldt.getPartitionKeys().size() != newt.getPartitionKeys().size() || !partKeysPartiallyEqual) {
                throw new InvalidOperationException("partition keys can not be changed.");
            }
        }
        // 4) the table was not initially created with a specified location
        if (rename && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) {
            Database olddb = msdb.getDatabase(dbname);
            // if a table was created in a user specified location using the DDL like
            // create table tbl ... location ...., it should be treated like an external table
            // in the table rename, its data location should not be changed. We can check
            // if the table directory was created directly under its database directory to tell
            // if it is such a table
            srcPath = new Path(oldt.getSd().getLocation());
            String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()).relativize(srcPath.toUri()).toString();
            boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR);
            if (!tableInSpecifiedLoc) {
                srcFs = wh.getFs(srcPath);
                // get new location
                Database db = msdb.getDatabase(newt.getDbName());
                Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
                destPath = new Path(databasePath, newt.getTableName().toLowerCase());
                destFs = wh.getFs(destPath);
                newt.getSd().setLocation(destPath.toString());
                // check that src and dest are on the same file system
                if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
                    throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported");
                }
                try {
                    if (destFs.exists(destPath)) {
                        throw new InvalidOperationException("New location for this table " + newt.getDbName() + "." + newt.getTableName() + " already exists : " + destPath);
                    }
                    // check that src exists and also checks permissions necessary, rename src to dest
                    if (srcFs.exists(srcPath) && srcFs.rename(srcPath, destPath)) {
                        dataWasMoved = true;
                    }
                } catch (IOException e) {
                    LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
                    throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
                }
                String oldTblLocPath = srcPath.toUri().getPath();
                String newTblLocPath = destPath.toUri().getPath();
                // also the location field in partition
                List<Partition> parts = msdb.getPartitions(dbname, name, -1);
                for (Partition part : parts) {
                    String oldPartLoc = part.getSd().getLocation();
                    if (oldPartLoc.contains(oldTblLocPath)) {
                        URI oldUri = new Path(oldPartLoc).toUri();
                        String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
                        Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
                        part.getSd().setLocation(newPartLocPath.toString());
                        String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
                        try {
                            //existing partition column stats is no longer valid, remove them
                            msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
                        } catch (InvalidInputException iie) {
                            throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
                        }
                        msdb.alterPartition(dbname, name, part.getValues(), part);
                    }
                }
            }
        } else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && (newt.getPartitionKeysSize() == 0)) {
            Database db = msdb.getDatabase(newt.getDbName());
            // Update table stats. For partitioned table, we update stats in
            // alterPartition()
            MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext);
        }
        alterTableUpdateTableColumnStats(msdb, oldt, newt);
        if (transactionalListeners != null && transactionalListeners.size() > 0) {
            AlterTableEvent alterTableEvent = new AlterTableEvent(oldt, newt, true, handler);
            alterTableEvent.setEnvironmentContext(environmentContext);
            for (MetaStoreEventListener transactionalListener : transactionalListeners) {
                transactionalListener.onAlterTable(alterTableEvent);
            }
        }
        // commit the changes
        success = msdb.commitTransaction();
    } catch (InvalidObjectException e) {
        LOG.debug("Failed to get object from Metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
    } catch (NoSuchObjectException e) {
        LOG.debug("Object not found in metastore ", e);
        throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
    } finally {
        if (!success) {
            LOG.error("Failed to alter table " + dbname + "." + name);
            msdb.rollbackTransaction();
            if (dataWasMoved) {
                try {
                    if (destFs.exists(destPath)) {
                        if (!destFs.rename(destPath, srcPath)) {
                            LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                        }
                    }
                } catch (IOException e) {
                    LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
                }
            }
        }
    }
}

Example 4 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class Hive method loadPartition.

/**
   * Load a directory into a Hive Table Partition - Alters existing content of
   * the partition with the contents of loadPath. - If the partition does not
   * exist - one is created - files in loadPath are moved into Hive. But the
   * directory itself is not removed.
   *
   * @param loadPath
   *          Directory containing files to load into Table
   * @param  tbl
   *          name of table to be loaded.
   * @param partSpec
   *          defines which partition needs to be loaded
   * @param replace
   *          if true - replace files in the partition, otherwise add files to
   *          the partition
   * @param inheritTableSpecs if true, on [re]creating the partition, take the
   *          location/inputformat/outputformat/serde details from table spec
   * @param isSrcLocal
   *          If the source directory is LOCAL
   * @param isAcid true if this is an ACID operation
   */
public Partition loadPartition(Path loadPath, Table tbl, Map<String, String> partSpec, boolean replace, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException {
    Path tblDataLocationPath = tbl.getDataLocation();
    try {
        Partition oldPart = getPartition(tbl, partSpec, false);
        /**
       * Move files before creating the partition since down stream processes
       * check for existence of partition in metadata before accessing the data.
       * If partition is created before data is moved, downstream waiting
       * processes might move forward with partial data
       */
        Path oldPartPath = (oldPart != null) ? oldPart.getDataLocation() : null;
        Path newPartPath = null;
        if (inheritTableSpecs) {
            Path partPath = new Path(tbl.getDataLocation(), Warehouse.makePartPath(partSpec));
            newPartPath = new Path(tblDataLocationPath.toUri().getScheme(), tblDataLocationPath.toUri().getAuthority(), partPath.toUri().getPath());
            if (oldPart != null) {
                /*
           * If we are moving the partition across filesystem boundaries
           * inherit from the table properties. Otherwise (same filesystem) use the
           * original partition location.
           *
           * See: HIVE-1707 and HIVE-2117 for background
           */
                FileSystem oldPartPathFS = oldPartPath.getFileSystem(getConf());
                FileSystem loadPathFS = loadPath.getFileSystem(getConf());
                if (FileUtils.equalsFileSystem(oldPartPathFS, loadPathFS)) {
                    newPartPath = oldPartPath;
                }
            }
        } else {
            newPartPath = oldPartPath;
        }
        List<Path> newFiles = null;
        PerfLogger perfLogger = SessionState.getPerfLogger();
        perfLogger.PerfLogBegin("MoveTask", "FileMoves");
        if (replace || (oldPart == null && !isAcid)) {
            replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), isSrcLocal);
        } else {
            if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPart != null) {
                newFiles = Collections.synchronizedList(new ArrayList<Path>());
            }
            FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
            Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, newFiles);
        }
        perfLogger.PerfLogEnd("MoveTask", "FileMoves");
        Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
        alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString());
        validatePartition(newTPart);
        if ((null != newFiles) || replace) {
            fireInsertEvent(tbl, partSpec, newFiles);
        } else {
            LOG.debug("No new files were created, and is not a replace. Skipping generating INSERT event.");
        }
        //column stats will be inaccurate
        StatsSetupConst.clearColumnStatsState(newTPart.getParameters());
        // recreate the partition if it existed before
        if (isSkewedStoreAsSubdir) {
            org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition();
            SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo();
            /* Construct list bucketing location mappings from sub-directory name. */
            Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap(newPartPath, skewedInfo);
            /* Add list bucketing location mappings. */
            skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
            newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
        }
        if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
            StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE);
        }
        if (oldPart == null) {
            newTPart.getTPartition().setParameters(new HashMap<String, String>());
            if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
                StatsSetupConst.setBasicStatsStateForCreateTable(newTPart.getParameters(), StatsSetupConst.TRUE);
            }
            MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters());
            try {
                LOG.debug("Adding new partition " + newTPart.getSpec());
                getSychronizedMSC().add_partition(newTPart.getTPartition());
            } catch (AlreadyExistsException aee) {
                // With multiple users concurrently issuing insert statements on the same partition has
                // a side effect that some queries may not see a partition at the time when they're issued,
                // but will realize the partition is actually there when it is trying to add such partition
                // to the metastore and thus get AlreadyExistsException, because some earlier query just created it (race condition).
                // For example, imagine such a table is created:
                //  create table T (name char(50)) partitioned by (ds string);
                // and the following two queries are launched at the same time, from different sessions:
                //  insert into table T partition (ds) values ('Bob', 'today'); -- creates the partition 'today'
                //  insert into table T partition (ds) values ('Joe', 'today'); -- will fail with AlreadyExistsException
                // In that case, we want to retry with alterPartition.
                LOG.debug("Caught AlreadyExistsException, trying to alter partition instead");
                setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart);
            }
        } else {
            setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart);
        }
        return newTPart;
    } catch (IOException e) {
        LOG.error(StringUtils.stringifyException(e));
        throw new HiveException(e);
    } catch (MetaException e) {
        LOG.error(StringUtils.stringifyException(e));
        throw new HiveException(e);
    } catch (InvalidOperationException e) {
        LOG.error(StringUtils.stringifyException(e));
        throw new HiveException(e);
    } catch (TException e) {
        LOG.error(StringUtils.stringifyException(e));
        throw new HiveException(e);
    }
}

Example 5 with InvalidOperationException

use of org.apache.hadoop.hive.metastore.api.InvalidOperationException in project hive by apache.

the class DDLTask method unarchive.

private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc) throws HiveException, URISyntaxException {
    Table tbl = db.getTable(simpleDesc.getTableName());
    // Means user specified a table, not a partition
    if (simpleDesc.getPartSpec() == null) {
        throw new HiveException("UNARCHIVE is for partitions only");
    }
    if (tbl.getTableType() != TableType.MANAGED_TABLE) {
        throw new HiveException("UNARCHIVE can only be performed on managed tables");
    }
    Map<String, String> partSpec = simpleDesc.getPartSpec();
    PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
    List<Partition> partitions = db.getPartitions(tbl, partSpec);
    int partSpecLevel = partSpec.size();
    Path originalDir = null;
    // to keep backward compatibility
    if (partitions.isEmpty()) {
        throw new HiveException("No partition matches the specification");
    } else if (partSpecInfo.values.size() != tbl.getPartCols().size()) {
        // for partial specifications we need partitions to follow the scheme
        for (Partition p : partitions) {
            if (partitionInCustomLocation(tbl, p)) {
                String message = String.format("UNARCHIVE cannot run for partition " + "groups with custom locations like %s", p.getLocation());
                throw new HiveException(message);
            }
        }
        originalDir = partSpecInfo.createPath(tbl);
    } else {
        Partition p = partitions.get(0);
        if (ArchiveUtils.isArchived(p)) {
            originalDir = new Path(getOriginalLocation(p));
        } else {
            originalDir = new Path(p.getLocation());
        }
    }
    URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
    Path intermediateArchivedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
    Path intermediateExtractedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
    boolean recovery = false;
    if (pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) {
        recovery = true;
        console.printInfo("Starting recovery after failed UNARCHIVE");
    }
    for (Partition p : partitions) {
        checkArchiveProperty(partSpecLevel, recovery, p);
    }
    String archiveName = "data.har";
    FileSystem fs = null;
    try {
        fs = originalDir.getFileSystem(conf);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    // assume the archive is in the original dir, check if it exists
    Path archivePath = new Path(originalDir, archiveName);
    URI archiveUri = archivePath.toUri();
    ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf, archiveUri, originalUri);
    URI sourceUri = harHelper.getHarUri(originalUri);
    Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath());
    if (!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) {
        throw new HiveException("Haven't found any archive where it should be");
    }
    Path tmpPath = driverContext.getCtx().getExternalTmpPath(originalDir);
    try {
        fs = tmpPath.getFileSystem(conf);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    if (!pathExists(intermediateExtractedDir) && !pathExists(intermediateArchivedDir)) {
        try {
            // Copy the files out of the archive into the temporary directory
            String copySource = sourceDir.toString();
            String copyDest = tmpPath.toString();
            List<String> args = new ArrayList<String>();
            args.add("-cp");
            args.add(copySource);
            args.add(copyDest);
            console.printInfo("Copying " + copySource + " to " + copyDest);
            FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
            srcFs.initialize(sourceDir.toUri(), conf);
            FsShell fss = new FsShell(conf);
            int ret = 0;
            try {
                ret = ToolRunner.run(fss, args.toArray(new String[0]));
            } catch (Exception e) {
                e.printStackTrace();
                throw new HiveException(e);
            }
            if (ret != 0) {
                throw new HiveException("Error while copying files from archive, return code=" + ret);
            } else {
                console.printInfo("Successfully Copied " + copySource + " to " + copyDest);
            }
            console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir);
            if (fs.exists(intermediateExtractedDir)) {
                throw new HiveException("Invalid state: the intermediate extracted " + "directory already exists.");
            }
            fs.rename(tmpPath, intermediateExtractedDir);
        } catch (Exception e) {
            throw new HiveException(e);
        }
    }
    if (!pathExists(intermediateArchivedDir)) {
        try {
            console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir);
            fs.rename(originalDir, intermediateArchivedDir);
        } catch (IOException e) {
            throw new HiveException(e);
        }
    } else {
        console.printInfo(intermediateArchivedDir + " already exists. " + "Assuming it contains the archived version of the partition");
    }
    // (containing the archived version of the files) to intermediateArchiveDir
    if (!pathExists(originalDir)) {
        try {
            console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir);
            fs.rename(intermediateExtractedDir, originalDir);
        } catch (IOException e) {
            throw new HiveException(e);
        }
    } else {
        console.printInfo(originalDir + " already exists. " + "Assuming it contains the extracted files in the partition");
    }
    for (Partition p : partitions) {
        setUnArchived(p);
        try {
            db.alterPartition(simpleDesc.getTableName(), p, null);
        } catch (InvalidOperationException e) {
            throw new HiveException(e);
        }
    }
    // deleted. The user will need to call unarchive again to clear those up.
    if (pathExists(intermediateArchivedDir)) {
        deleteDir(intermediateArchivedDir);
    }
    if (recovery) {
        console.printInfo("Recovery after UNARCHIVE succeeded");
    }
    return 0;
}