use of org.apache.hadoop.hive.metastore.api.InvalidInputException in project hive by apache.
the class HiveAlterHandler method updatePartColumnStatsForAlterColumns.
private void updatePartColumnStatsForAlterColumns(RawStore msdb, Partition oldPartition, String oldPartName, List<String> partVals, List<FieldSchema> oldCols, Partition newPart) throws MetaException, InvalidObjectException {
String dbName = oldPartition.getDbName();
String tableName = oldPartition.getTableName();
try {
List<String> oldPartNames = Lists.newArrayList(oldPartName);
List<String> oldColNames = new ArrayList<String>(oldCols.size());
for (FieldSchema oldCol : oldCols) {
oldColNames.add(oldCol.getName());
}
List<FieldSchema> newCols = newPart.getSd().getCols();
List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(dbName, tableName, oldPartNames, oldColNames);
assert (partsColStats.size() <= 1);
for (ColumnStatistics partColStats : partsColStats) {
//actually only at most one loop
List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj();
List<String> deletedCols = new ArrayList<String>();
for (ColumnStatisticsObj statsObj : statsObjs) {
boolean found = false;
for (FieldSchema newCol : newCols) {
if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
found = true;
break;
}
}
if (!found) {
msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, statsObj.getColName());
deletedCols.add(statsObj.getColName());
}
}
StatsSetupConst.removeColumnStatsState(newPart.getParameters(), deletedCols);
}
} catch (NoSuchObjectException nsoe) {
LOG.debug("Could not find db entry." + nsoe);
//ignore
} catch (InvalidInputException iie) {
throw new InvalidObjectException("Invalid input to update partition column stats in alter table change columns" + iie);
}
}
use of org.apache.hadoop.hive.metastore.api.InvalidInputException in project hive by apache.
the class HiveAlterHandler method alterPartition.
@Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
boolean success = false;
Path srcPath = null;
Path destPath = null;
FileSystem srcFs = null;
FileSystem destFs;
Partition oldPart = null;
String oldPartLoc = null;
String newPartLoc = null;
List<MetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
// Set DDL time to now if not specified
if (new_part.getParameters() == null || new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
}
Table tbl = msdb.getTable(dbname, name);
if (tbl == null) {
throw new InvalidObjectException("Unable to alter partition because table or database does not exist.");
}
//alter partition
if (part_vals == null || part_vals.size() == 0) {
try {
msdb.openTransaction();
oldPart = msdb.getPartition(dbname, name, new_part.getValues());
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
// if stats are same, no need to update
if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
} else {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
}
}
updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
throw new InvalidOperationException("alter is not possible");
} catch (NoSuchObjectException e) {
//old partition does not exist
throw new InvalidOperationException("alter is not possible");
} finally {
if (!success) {
msdb.rollbackTransaction();
}
}
return oldPart;
}
//rename partition
try {
msdb.openTransaction();
try {
oldPart = msdb.getPartition(dbname, name, part_vals);
} catch (NoSuchObjectException e) {
// this means there is no existing partition
throw new InvalidObjectException("Unable to rename partition because old partition does not exist");
}
Partition check_part;
try {
check_part = msdb.getPartition(dbname, name, new_part.getValues());
} catch (NoSuchObjectException e) {
// this means there is no existing partition
check_part = null;
}
if (check_part != null) {
throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + new_part.getValues());
}
// if the external partition is renamed, the file should not change
if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
new_part.getSd().setLocation(oldPart.getSd().getLocation());
String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
try {
//existing partition column stats is no longer valid, remove
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
} catch (NoSuchObjectException nsoe) {
//ignore
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part_vals, new_part);
} else {
try {
destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
} catch (NoSuchObjectException e) {
LOG.debug("Didn't find object in metastore ", e);
throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
}
if (destPath != null) {
newPartLoc = destPath.toString();
oldPartLoc = oldPart.getSd().getLocation();
LOG.info("srcPath:" + oldPartLoc);
LOG.info("descPath:" + newPartLoc);
srcPath = new Path(oldPartLoc);
srcFs = wh.getFs(srcPath);
destFs = wh.getFs(destPath);
// check that src and dest are on the same file system
if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("New table location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported.");
}
try {
srcFs.exists(srcPath);
if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
throw new InvalidOperationException("New location for this table " + tbl.getDbName() + "." + tbl.getTableName() + " already exists : " + destPath);
}
} catch (IOException e) {
throw new InvalidOperationException("Unable to access new location " + destPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
}
new_part.getSd().setLocation(newPartLoc);
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
}
String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
try {
//existing partition column stats is no longer valid, remove
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
} catch (NoSuchObjectException nsoe) {
//ignore
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part_vals, new_part);
}
}
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
success = msdb.commitTransaction();
} finally {
if (!success) {
msdb.rollbackTransaction();
}
if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
//rename the data directory
try {
if (srcFs.exists(srcPath)) {
//if destPath's parent path doesn't exist, we should mkdir it
Path destParentPath = destPath.getParent();
if (!wh.mkdirs(destParentPath, true)) {
throw new IOException("Unable to create path " + destParentPath);
}
wh.renameDir(srcPath, destPath, true);
LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
}
} catch (IOException ex) {
LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, ex);
boolean revertMetaDataTransaction = false;
try {
msdb.openTransaction();
msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(new_part, oldPart, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
revertMetaDataTransaction = msdb.commitTransaction();
} catch (Exception ex2) {
LOG.error("Attempt to revert partition metadata change failed. The revert was attempted " + "because associated filesystem rename operation failed with exception " + ex.getMessage(), ex2);
if (!revertMetaDataTransaction) {
msdb.rollbackTransaction();
}
}
throw new InvalidOperationException("Unable to access old location " + srcPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + part_vals);
}
}
}
return oldPart;
}
use of org.apache.hadoop.hive.metastore.api.InvalidInputException in project hive by apache.
the class HiveAlterHandler method alterTable.
@Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, MetaException {
final boolean cascade = environmentContext != null && environmentContext.isSetProperties() && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE));
if (newt == null) {
throw new InvalidOperationException("New table is invalid: " + newt);
}
if (!MetaStoreUtils.validateName(newt.getTableName(), hiveConf)) {
throw new InvalidOperationException(newt.getTableName() + " is not a valid object name");
}
String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
if (validate != null) {
throw new InvalidOperationException("Invalid column " + validate);
}
Path srcPath = null;
FileSystem srcFs = null;
Path destPath = null;
FileSystem destFs = null;
boolean success = false;
boolean dataWasMoved = false;
boolean rename = false;
Table oldt = null;
List<MetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
try {
msdb.openTransaction();
name = name.toLowerCase();
dbname = dbname.toLowerCase();
// check if table with the new name already exists
if (!newt.getTableName().equalsIgnoreCase(name) || !newt.getDbName().equalsIgnoreCase(dbname)) {
if (msdb.getTable(newt.getDbName(), newt.getTableName()) != null) {
throw new InvalidOperationException("new table " + newt.getDbName() + "." + newt.getTableName() + " already exists");
}
rename = true;
}
// get old table
oldt = msdb.getTable(dbname, name);
if (oldt == null) {
throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
}
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, false)) {
// Throws InvalidOperationException if the new column types are not
// compatible with the current column types.
MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(oldt.getSd().getCols(), newt.getSd().getCols());
}
if (cascade) {
//Currently only column related changes can be cascaded in alter table
if (MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
List<FieldSchema> oldCols = part.getSd().getCols();
part.getSd().setCols(newt.getSd().getCols());
String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
msdb.alterPartition(dbname, name, part.getValues(), part);
}
} else {
LOG.warn("Alter table does not cascade changes to its partitions.");
}
}
//check that partition keys have not changed, except for virtual views
//however, allow the partition comments to change
boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), newt.getPartitionKeys());
if (!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
if (oldt.getPartitionKeys().size() != newt.getPartitionKeys().size() || !partKeysPartiallyEqual) {
throw new InvalidOperationException("partition keys can not be changed.");
}
}
// 4) the table was not initially created with a specified location
if (rename && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) {
Database olddb = msdb.getDatabase(dbname);
// if a table was created in a user specified location using the DDL like
// create table tbl ... location ...., it should be treated like an external table
// in the table rename, its data location should not be changed. We can check
// if the table directory was created directly under its database directory to tell
// if it is such a table
srcPath = new Path(oldt.getSd().getLocation());
String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()).relativize(srcPath.toUri()).toString();
boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR);
if (!tableInSpecifiedLoc) {
srcFs = wh.getFs(srcPath);
// get new location
Database db = msdb.getDatabase(newt.getDbName());
Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
destPath = new Path(databasePath, newt.getTableName().toLowerCase());
destFs = wh.getFs(destPath);
newt.getSd().setLocation(destPath.toString());
// check that src and dest are on the same file system
if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported");
}
try {
if (destFs.exists(destPath)) {
throw new InvalidOperationException("New location for this table " + newt.getDbName() + "." + newt.getTableName() + " already exists : " + destPath);
}
// check that src exists and also checks permissions necessary, rename src to dest
if (srcFs.exists(srcPath) && srcFs.rename(srcPath, destPath)) {
dataWasMoved = true;
}
} catch (IOException e) {
LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
}
String oldTblLocPath = srcPath.toUri().getPath();
String newTblLocPath = destPath.toUri().getPath();
// also the location field in partition
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
String oldPartLoc = part.getSd().getLocation();
if (oldPartLoc.contains(oldTblLocPath)) {
URI oldUri = new Path(oldPartLoc).toUri();
String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
part.getSd().setLocation(newPartLocPath.toString());
String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
try {
//existing partition column stats is no longer valid, remove them
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part.getValues(), part);
}
}
}
} else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && (newt.getPartitionKeysSize() == 0)) {
Database db = msdb.getDatabase(newt.getDbName());
// Update table stats. For partitioned table, we update stats in
// alterPartition()
MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext);
}
alterTableUpdateTableColumnStats(msdb, oldt, newt);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterTableEvent alterTableEvent = new AlterTableEvent(oldt, newt, true, handler);
alterTableEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterTable(alterTableEvent);
}
}
// commit the changes
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
LOG.debug("Failed to get object from Metastore ", e);
throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
} catch (NoSuchObjectException e) {
LOG.debug("Object not found in metastore ", e);
throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
} finally {
if (!success) {
LOG.error("Failed to alter table " + dbname + "." + name);
msdb.rollbackTransaction();
if (dataWasMoved) {
try {
if (destFs.exists(destPath)) {
if (!destFs.rename(destPath, srcPath)) {
LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
}
}
} catch (IOException e) {
LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
}
}
}
}
}
use of org.apache.hadoop.hive.metastore.api.InvalidInputException in project hive by apache.
the class HiveAlterHandler method updatePartColumnStats.
private void updatePartColumnStats(RawStore msdb, String dbName, String tableName, List<String> partVals, Partition newPart) throws MetaException, InvalidObjectException {
dbName = HiveStringUtils.normalizeIdentifier(dbName);
tableName = HiveStringUtils.normalizeIdentifier(tableName);
String newDbName = HiveStringUtils.normalizeIdentifier(newPart.getDbName());
String newTableName = HiveStringUtils.normalizeIdentifier(newPart.getTableName());
Table oldTable = msdb.getTable(dbName, tableName);
if (oldTable == null) {
return;
}
try {
String oldPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), partVals);
String newPartName = Warehouse.makePartName(oldTable.getPartitionKeys(), newPart.getValues());
if (!dbName.equals(newDbName) || !tableName.equals(newTableName) || !oldPartName.equals(newPartName)) {
msdb.deletePartitionColumnStatistics(dbName, tableName, oldPartName, partVals, null);
} else {
Partition oldPartition = msdb.getPartition(dbName, tableName, partVals);
if (oldPartition == null) {
return;
}
if (oldPartition.getSd() != null && newPart.getSd() != null) {
List<FieldSchema> oldCols = oldPartition.getSd().getCols();
if (!MetaStoreUtils.columnsIncluded(oldCols, newPart.getSd().getCols())) {
updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals, oldCols, newPart);
}
}
}
} catch (NoSuchObjectException nsoe) {
LOG.debug("Could not find db entry." + nsoe);
//ignore
} catch (InvalidInputException iie) {
throw new InvalidObjectException("Invalid input to update partition column stats." + iie);
}
}
use of org.apache.hadoop.hive.metastore.api.InvalidInputException in project hive by apache.
the class HiveAlterHandler method alterTableUpdateTableColumnStats.
@VisibleForTesting
void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable) throws MetaException, InvalidObjectException {
String dbName = oldTable.getDbName().toLowerCase();
String tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(oldTable.getTableName());
String newDbName = newTable.getDbName().toLowerCase();
String newTableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(newTable.getTableName());
try {
List<FieldSchema> oldCols = oldTable.getSd().getCols();
List<FieldSchema> newCols = newTable.getSd().getCols();
List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>();
ColumnStatistics colStats = null;
boolean updateColumnStats = true;
// Nothing to update if everything is the same
if (newDbName.equals(dbName) && newTableName.equals(tableName) && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) {
updateColumnStats = false;
}
if (updateColumnStats) {
List<String> oldColNames = new ArrayList<>(oldCols.size());
for (FieldSchema oldCol : oldCols) {
oldColNames.add(oldCol.getName());
}
// Collect column stats which need to be rewritten and remove old stats
colStats = msdb.getTableColumnStatistics(dbName, tableName, oldColNames);
if (colStats == null) {
updateColumnStats = false;
} else {
List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
if (statsObjs != null) {
List<String> deletedCols = new ArrayList<>();
for (ColumnStatisticsObj statsObj : statsObjs) {
boolean found = false;
for (FieldSchema newCol : newCols) {
if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) && statsObj.getColType().equalsIgnoreCase(newCol.getType())) {
found = true;
break;
}
}
if (found) {
if (!newDbName.equals(dbName) || !newTableName.equals(tableName)) {
msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
newStatsObjs.add(statsObj);
deletedCols.add(statsObj.getColName());
}
} else {
msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
deletedCols.add(statsObj.getColName());
}
}
StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols);
}
}
}
// Change to new table and append stats for the new table
msdb.alterTable(dbName, tableName, newTable);
if (updateColumnStats && !newStatsObjs.isEmpty()) {
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
statsDesc.setDbName(newDbName);
statsDesc.setTableName(newTableName);
colStats.setStatsObj(newStatsObjs);
msdb.updateTableColumnStatistics(colStats);
}
} catch (NoSuchObjectException nsoe) {
LOG.debug("Could not find db entry." + nsoe);
} catch (InvalidInputException e) {
// should not happen since the input were verified before passed in
throw new InvalidObjectException("Invalid inputs to update table column stats: " + e);
}
}
Aggregations