use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.
the class HiveAlterHandler method alterTable.
@Override
public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, MetaException {
final boolean cascade = environmentContext != null && environmentContext.isSetProperties() && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(StatsSetupConst.CASCADE));
if (newt == null) {
throw new InvalidOperationException("New table is invalid: " + newt);
}
if (!MetaStoreUtils.validateName(newt.getTableName(), hiveConf)) {
throw new InvalidOperationException(newt.getTableName() + " is not a valid object name");
}
String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols());
if (validate != null) {
throw new InvalidOperationException("Invalid column " + validate);
}
Path srcPath = null;
FileSystem srcFs = null;
Path destPath = null;
FileSystem destFs = null;
boolean success = false;
boolean dataWasMoved = false;
boolean rename = false;
Table oldt = null;
List<MetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
try {
msdb.openTransaction();
name = name.toLowerCase();
dbname = dbname.toLowerCase();
// check if table with the new name already exists
if (!newt.getTableName().equalsIgnoreCase(name) || !newt.getDbName().equalsIgnoreCase(dbname)) {
if (msdb.getTable(newt.getDbName(), newt.getTableName()) != null) {
throw new InvalidOperationException("new table " + newt.getDbName() + "." + newt.getTableName() + " already exists");
}
rename = true;
}
// get old table
oldt = msdb.getTable(dbname, name);
if (oldt == null) {
throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
}
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, false)) {
// Throws InvalidOperationException if the new column types are not
// compatible with the current column types.
MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(oldt.getSd().getCols(), newt.getSd().getCols());
}
if (cascade) {
//Currently only column related changes can be cascaded in alter table
if (MetaStoreUtils.isCascadeNeededInAlterTable(oldt, newt)) {
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
List<FieldSchema> oldCols = part.getSd().getCols();
part.getSd().setCols(newt.getSd().getCols());
String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part);
msdb.alterPartition(dbname, name, part.getValues(), part);
}
} else {
LOG.warn("Alter table does not cascade changes to its partitions.");
}
}
//check that partition keys have not changed, except for virtual views
//however, allow the partition comments to change
boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), newt.getPartitionKeys());
if (!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) {
if (oldt.getPartitionKeys().size() != newt.getPartitionKeys().size() || !partKeysPartiallyEqual) {
throw new InvalidOperationException("partition keys can not be changed.");
}
}
// 4) the table was not initially created with a specified location
if (rename && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 || StringUtils.isEmpty(newt.getSd().getLocation())) && !MetaStoreUtils.isExternalTable(oldt)) {
Database olddb = msdb.getDatabase(dbname);
// if a table was created in a user specified location using the DDL like
// create table tbl ... location ...., it should be treated like an external table
// in the table rename, its data location should not be changed. We can check
// if the table directory was created directly under its database directory to tell
// if it is such a table
srcPath = new Path(oldt.getSd().getLocation());
String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()).relativize(srcPath.toUri()).toString();
boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR);
if (!tableInSpecifiedLoc) {
srcFs = wh.getFs(srcPath);
// get new location
Database db = msdb.getDatabase(newt.getDbName());
Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
destPath = new Path(databasePath, newt.getTableName().toLowerCase());
destFs = wh.getFs(destPath);
newt.getSd().setLocation(destPath.toString());
// check that src and dest are on the same file system
if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported");
}
try {
if (destFs.exists(destPath)) {
throw new InvalidOperationException("New location for this table " + newt.getDbName() + "." + newt.getTableName() + " already exists : " + destPath);
}
// check that src exists and also checks permissions necessary, rename src to dest
if (srcFs.exists(srcPath) && srcFs.rename(srcPath, destPath)) {
dataWasMoved = true;
}
} catch (IOException e) {
LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
}
String oldTblLocPath = srcPath.toUri().getPath();
String newTblLocPath = destPath.toUri().getPath();
// also the location field in partition
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
String oldPartLoc = part.getSd().getLocation();
if (oldPartLoc.contains(oldTblLocPath)) {
URI oldUri = new Path(oldPartLoc).toUri();
String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
part.getSd().setLocation(newPartLocPath.toString());
String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues());
try {
//existing partition column stats is no longer valid, remove them
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, part.getValues(), null);
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part.getValues(), part);
}
}
}
} else if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && (newt.getPartitionKeysSize() == 0)) {
Database db = msdb.getDatabase(newt.getDbName());
// Update table stats. For partitioned table, we update stats in
// alterPartition()
MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext);
}
alterTableUpdateTableColumnStats(msdb, oldt, newt);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterTableEvent alterTableEvent = new AlterTableEvent(oldt, newt, true, handler);
alterTableEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterTable(alterTableEvent);
}
}
// commit the changes
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
LOG.debug("Failed to get object from Metastore ", e);
throw new InvalidOperationException("Unable to change partition or table." + " Check metastore logs for detailed stack." + e.getMessage());
} catch (NoSuchObjectException e) {
LOG.debug("Object not found in metastore ", e);
throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
} finally {
if (!success) {
LOG.error("Failed to alter table " + dbname + "." + name);
msdb.rollbackTransaction();
if (dataWasMoved) {
try {
if (destFs.exists(destPath)) {
if (!destFs.rename(destPath, srcPath)) {
LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
}
}
} catch (IOException e) {
LOG.error("Failed to restore data from " + destPath + " to " + srcPath + " in alter table failure. Manual restore is needed.");
}
}
}
}
}
use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.
the class TestHiveMetaStoreWithEnvironmentContext method testEnvironmentContext.
public void testEnvironmentContext() throws Exception {
int listSize = 0;
List<ListenerEvent> notifyList = DummyListener.notifyList;
assertEquals(notifyList.size(), listSize);
msc.createDatabase(db);
listSize++;
assertEquals(listSize, notifyList.size());
CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
assert dbEvent.getStatus();
msc.createTable(table, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
assert tblEvent.getStatus();
assertEquals(envContext, tblEvent.getEnvironmentContext());
table = msc.getTable(dbName, tblName);
partition.getSd().setLocation(table.getSd().getLocation() + "/part1");
msc.add_partition(partition, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assert partEvent.getStatus();
assertEquals(envContext, partEvent.getEnvironmentContext());
List<String> partVals = new ArrayList<String>();
partVals.add("2012");
msc.appendPartition(dbName, tblName, partVals, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assert appendPartEvent.getStatus();
assertEquals(envContext, appendPartEvent.getEnvironmentContext());
table.setTableName(renamed);
msc.alter_table_with_environmentContext(dbName, tblName, table, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
AlterTableEvent alterTableEvent = (AlterTableEvent) notifyList.get(listSize - 1);
assert alterTableEvent.getStatus();
assertEquals(envContext, alterTableEvent.getEnvironmentContext());
table.setTableName(tblName);
msc.alter_table_with_environmentContext(dbName, renamed, table, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
List<String> dropPartVals = new ArrayList<String>();
dropPartVals.add("2011");
msc.dropPartition(dbName, tblName, dropPartVals, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
DropPartitionEvent dropPartEvent = (DropPartitionEvent) notifyList.get(listSize - 1);
assert dropPartEvent.getStatus();
assertEquals(envContext, dropPartEvent.getEnvironmentContext());
msc.dropPartition(dbName, tblName, "b=2012", true, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
DropPartitionEvent dropPartByNameEvent = (DropPartitionEvent) notifyList.get(listSize - 1);
assert dropPartByNameEvent.getStatus();
assertEquals(envContext, dropPartByNameEvent.getEnvironmentContext());
msc.dropTable(dbName, tblName, true, false, envContext);
listSize++;
assertEquals(notifyList.size(), listSize);
DropTableEvent dropTblEvent = (DropTableEvent) notifyList.get(listSize - 1);
assert dropTblEvent.getStatus();
assertEquals(envContext, dropTblEvent.getEnvironmentContext());
msc.dropDatabase(dbName);
listSize++;
assertEquals(notifyList.size(), listSize);
DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
assert dropDB.getStatus();
}
use of org.apache.hadoop.hive.metastore.events.AlterTableEvent in project hive by apache.
the class TestMetaStoreEventListener method testListener.
public void testListener() throws Exception {
int listSize = 0;
List<ListenerEvent> notifyList = DummyListener.notifyList;
List<PreEventContext> preNotifyList = DummyPreListener.notifyList;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), listSize);
driver.run("create database " + dbName);
listSize++;
PreCreateDatabaseEvent preDbEvent = (PreCreateDatabaseEvent) (preNotifyList.get(preNotifyList.size() - 1));
Database db = msc.getDatabase(dbName);
assertEquals(listSize, notifyList.size());
assertEquals(listSize + 1, preNotifyList.size());
validateCreateDb(db, preDbEvent.getDatabase());
CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
assert dbEvent.getStatus();
validateCreateDb(db, dbEvent.getDatabase());
driver.run("use " + dbName);
driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
PreCreateTableEvent preTblEvent = (PreCreateTableEvent) (preNotifyList.get(preNotifyList.size() - 1));
listSize++;
Table tbl = msc.getTable(dbName, tblName);
validateCreateTable(tbl, preTblEvent.getTable());
assertEquals(notifyList.size(), listSize);
CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
assert tblEvent.getStatus();
validateCreateTable(tbl, tblEvent.getTable());
driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')");
// creates index table internally
listSize += 2;
assertEquals(notifyList.size(), listSize);
AddIndexEvent addIndexEvent = (AddIndexEvent) notifyList.get(listSize - 1);
assert addIndexEvent.getStatus();
PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent) (preNotifyList.get(preNotifyList.size() - 3));
Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
validateAddIndex(oldIndex, addIndexEvent.getIndex());
validateAddIndex(oldIndex, preAddIndexEvent.getIndex());
driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + "('prop1'='val1_new', 'prop3'='val3')");
listSize++;
assertEquals(notifyList.size(), listSize);
Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1);
assert alterIndexEvent.getStatus();
validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), newIndex, alterIndexEvent.getNewIndex());
PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), newIndex, preAlterIndexEvent.getNewIndex());
driver.run("drop index tmptbl_i on tmptbl");
listSize++;
assertEquals(notifyList.size(), listSize);
DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1);
assert dropIndexEvent.getStatus();
validateDropIndex(newIndex, dropIndexEvent.getIndex());
PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateDropIndex(newIndex, preDropIndexEvent.getIndex());
driver.run("alter table tmptbl add partition (b='2011')");
listSize++;
assertEquals(notifyList.size(), listSize);
PreAddPartitionEvent prePartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assert partEvent.getStatus();
Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
Partition partAdded = partEvent.getPartitionIterator().next();
validateAddPartition(part, partAdded);
validateTableInAddPartition(tbl, partEvent.getTable());
validateAddPartition(part, prePartEvent.getPartitions().get(0));
// Test adding multiple partitions in a single partition-set, atomically.
int currentTime = (int) System.currentTimeMillis();
HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(hiveConf);
Table table = hmsClient.getTable(dbName, "tmptbl");
Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition2 = new Partition(Arrays.asList("20110102"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition3 = new Partition(Arrays.asList("20110103"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
++listSize;
AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable());
List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterPartitionEvent preAlterPartEvent = (PreAlterPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
//the partition did not change,
// so the new partition should be similar to the original partition
Partition origP = msc.getPartition(dbName, tblName, "b=2011");
AlterPartitionEvent alterPartEvent = (AlterPartitionEvent) notifyList.get(listSize - 1);
assert alterPartEvent.getStatus();
validateAlterPartition(origP, origP, alterPartEvent.getOldPartition().getDbName(), alterPartEvent.getOldPartition().getTableName(), alterPartEvent.getOldPartition().getValues(), alterPartEvent.getNewPartition());
validateAlterPartition(origP, origP, preAlterPartEvent.getDbName(), preAlterPartEvent.getTableName(), preAlterPartEvent.getNewPartition().getValues(), preAlterPartEvent.getNewPartition());
List<String> part_vals = new ArrayList<String>();
part_vals.add("c=2012");
int preEventListSize;
preEventListSize = preNotifyList.size() + 1;
Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
listSize++;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), preEventListSize);
AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
Partition partAppended = appendPartEvent.getPartitionIterator().next();
validateAddPartition(newPart, partAppended);
PreAddPartitionEvent preAppendPartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateAddPartition(newPart, preAppendPartEvent.getPartitions().get(0));
driver.run(String.format("alter table %s rename to %s", tblName, renamed));
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterTableEvent preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
Table renamedTable = msc.getTable(dbName, renamed);
AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
assert alterTableE.getStatus();
validateAlterTable(tbl, renamedTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTable(tbl, renamedTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
//change the table name back
driver.run(String.format("alter table %s rename to %s", renamed, tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
Table altTable = msc.getTable(dbName, tblName);
alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
assert alterTableE.getStatus();
validateAlterTableColumns(tbl, altTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTableColumns(tbl, altTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
Map<String, String> kvs = new HashMap<String, String>(1);
kvs.put("b", "2011");
msc.markPartitionForEvent("hive2038", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
listSize++;
assertEquals(notifyList.size(), listSize);
LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent) notifyList.get(listSize - 1);
assert partMarkEvent.getStatus();
validateLoadPartitionDone("tmptbl", kvs, partMarkEvent.getTable().getTableName(), partMarkEvent.getPartitionName());
PreLoadPartitionDoneEvent prePartMarkEvent = (PreLoadPartitionDoneEvent) preNotifyList.get(preNotifyList.size() - 1);
validateLoadPartitionDone("tmptbl", kvs, prePartMarkEvent.getTableName(), prePartMarkEvent.getPartitionName());
driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropPartitionEvent preDropPart = (PreDropPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
DropPartitionEvent dropPart = (DropPartitionEvent) notifyList.get(listSize - 1);
assert dropPart.getStatus();
validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator());
validateTableInDropPartition(tbl, dropPart.getTable());
validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator());
validateTableInDropPartition(tbl, preDropPart.getTable());
driver.run("drop table " + tblName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropTableEvent preDropTbl = (PreDropTableEvent) preNotifyList.get(preNotifyList.size() - 1);
DropTableEvent dropTbl = (DropTableEvent) notifyList.get(listSize - 1);
assert dropTbl.getStatus();
validateDropTable(tbl, dropTbl.getTable());
validateDropTable(tbl, preDropTbl.getTable());
driver.run("drop database " + dbName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropDatabaseEvent preDropDB = (PreDropDatabaseEvent) preNotifyList.get(preNotifyList.size() - 1);
DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
assert dropDB.getStatus();
validateDropDb(db, dropDB.getDatabase());
validateDropDb(db, preDropDB.getDatabase());
SetProcessor.setVariable("metaconf:hive.metastore.try.direct.sql", "false");
ConfigChangeEvent event = (ConfigChangeEvent) notifyList.get(notifyList.size() - 1);
assertEquals("hive.metastore.try.direct.sql", event.getKey());
assertEquals("true", event.getOldValue());
assertEquals("false", event.getNewValue());
}
Aggregations