use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.
the class HiveAlterHandler method alterPartitions.
@Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<Partition> new_parts, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
List<Partition> oldParts = new ArrayList<Partition>();
List<List<String>> partValsList = new ArrayList<List<String>>();
List<MetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
Table tbl = msdb.getTable(dbname, name);
if (tbl == null) {
throw new InvalidObjectException("Unable to alter partitions because table or database does not exist.");
}
boolean success = false;
try {
msdb.openTransaction();
for (Partition tmpPart : new_parts) {
// Set DDL time to now if not specified
if (tmpPart.getParameters() == null || tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
tmpPart.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
}
Partition oldTmpPart = msdb.getPartition(dbname, name, tmpPart.getValues());
oldParts.add(oldTmpPart);
partValsList.add(tmpPart.getValues());
if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) {
// Check if stats are same, no need to update
if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());
} else {
MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
}
}
updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
}
msdb.alterPartitions(dbname, name, partValsList, new_parts);
Iterator<Partition> oldPartsIt = oldParts.iterator();
for (Partition newPart : new_parts) {
Partition oldPart;
if (oldPartsIt.hasNext()) {
oldPart = oldPartsIt.next();
} else {
throw new InvalidOperationException("Missing old partition corresponding to new partition " + "when invoking MetaStoreEventListener for alterPartitions event.");
}
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, newPart, tbl, true, handler);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
}
success = msdb.commitTransaction();
} catch (InvalidObjectException | NoSuchObjectException e) {
throw new InvalidOperationException("Alter partition operation failed: " + e);
} finally {
if (!success) {
msdb.rollbackTransaction();
}
}
return oldParts;
}
use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.
the class HiveAlterHandler method alterPartition.
@Override
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List<String> part_vals, final Partition new_part, EnvironmentContext environmentContext, HMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
boolean success = false;
Path srcPath = null;
Path destPath = null;
FileSystem srcFs = null;
FileSystem destFs;
Partition oldPart = null;
String oldPartLoc = null;
String newPartLoc = null;
List<MetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
// Set DDL time to now if not specified
if (new_part.getParameters() == null || new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System.currentTimeMillis() / 1000));
}
Table tbl = msdb.getTable(dbname, name);
if (tbl == null) {
throw new InvalidObjectException("Unable to alter partition because table or database does not exist.");
}
//alter partition
if (part_vals == null || part_vals.size() == 0) {
try {
msdb.openTransaction();
oldPart = msdb.getPartition(dbname, name, new_part.getValues());
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
// if stats are same, no need to update
if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
} else {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
}
}
updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
msdb.alterPartition(dbname, name, new_part.getValues(), new_part);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
throw new InvalidOperationException("alter is not possible");
} catch (NoSuchObjectException e) {
//old partition does not exist
throw new InvalidOperationException("alter is not possible");
} finally {
if (!success) {
msdb.rollbackTransaction();
}
}
return oldPart;
}
//rename partition
try {
msdb.openTransaction();
try {
oldPart = msdb.getPartition(dbname, name, part_vals);
} catch (NoSuchObjectException e) {
// this means there is no existing partition
throw new InvalidObjectException("Unable to rename partition because old partition does not exist");
}
Partition check_part;
try {
check_part = msdb.getPartition(dbname, name, new_part.getValues());
} catch (NoSuchObjectException e) {
// this means there is no existing partition
check_part = null;
}
if (check_part != null) {
throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + new_part.getValues());
}
// if the external partition is renamed, the file should not change
if (tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
new_part.getSd().setLocation(oldPart.getSd().getLocation());
String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
try {
//existing partition column stats is no longer valid, remove
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
} catch (NoSuchObjectException nsoe) {
//ignore
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part_vals, new_part);
} else {
try {
destPath = new Path(wh.getTablePath(msdb.getDatabase(dbname), name), Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()));
destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
} catch (NoSuchObjectException e) {
LOG.debug("Didn't find object in metastore ", e);
throw new InvalidOperationException("Unable to change partition or table. Database " + dbname + " does not exist" + " Check metastore logs for detailed stack." + e.getMessage());
}
if (destPath != null) {
newPartLoc = destPath.toString();
oldPartLoc = oldPart.getSd().getLocation();
LOG.info("srcPath:" + oldPartLoc);
LOG.info("descPath:" + newPartLoc);
srcPath = new Path(oldPartLoc);
srcFs = wh.getFs(srcPath);
destFs = wh.getFs(destPath);
// check that src and dest are on the same file system
if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("New table location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported.");
}
try {
srcFs.exists(srcPath);
if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
throw new InvalidOperationException("New location for this table " + tbl.getDbName() + "." + tbl.getTableName() + " already exists : " + destPath);
}
} catch (IOException e) {
throw new InvalidOperationException("Unable to access new location " + destPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
}
new_part.getSd().setLocation(newPartLoc);
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
}
String oldPartName = Warehouse.makePartName(tbl.getPartitionKeys(), oldPart.getValues());
try {
//existing partition column stats is no longer valid, remove
msdb.deletePartitionColumnStatistics(dbname, name, oldPartName, oldPart.getValues(), null);
} catch (NoSuchObjectException nsoe) {
//ignore
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
}
msdb.alterPartition(dbname, name, part_vals, new_part);
}
}
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(oldPart, new_part, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
success = msdb.commitTransaction();
} finally {
if (!success) {
msdb.rollbackTransaction();
}
if (success && newPartLoc != null && newPartLoc.compareTo(oldPartLoc) != 0) {
//rename the data directory
try {
if (srcFs.exists(srcPath)) {
//if destPath's parent path doesn't exist, we should mkdir it
Path destParentPath = destPath.getParent();
if (!wh.mkdirs(destParentPath, true)) {
throw new IOException("Unable to create path " + destParentPath);
}
wh.renameDir(srcPath, destPath, true);
LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
}
} catch (IOException ex) {
LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, ex);
boolean revertMetaDataTransaction = false;
try {
msdb.openTransaction();
msdb.alterPartition(dbname, name, new_part.getValues(), oldPart);
if (transactionalListeners != null && transactionalListeners.size() > 0) {
AlterPartitionEvent alterPartitionEvent = new AlterPartitionEvent(new_part, oldPart, tbl, true, handler);
alterPartitionEvent.setEnvironmentContext(environmentContext);
for (MetaStoreEventListener transactionalListener : transactionalListeners) {
transactionalListener.onAlterPartition(alterPartitionEvent);
}
}
revertMetaDataTransaction = msdb.commitTransaction();
} catch (Exception ex2) {
LOG.error("Attempt to revert partition metadata change failed. The revert was attempted " + "because associated filesystem rename operation failed with exception " + ex.getMessage(), ex2);
if (!revertMetaDataTransaction) {
msdb.rollbackTransaction();
}
}
throw new InvalidOperationException("Unable to access old location " + srcPath + " for partition " + tbl.getDbName() + "." + tbl.getTableName() + " " + part_vals);
}
}
}
return oldPart;
}
use of org.apache.hadoop.hive.metastore.events.AlterPartitionEvent in project hive by apache.
the class TestMetaStoreEventListener method testListener.
public void testListener() throws Exception {
int listSize = 0;
List<ListenerEvent> notifyList = DummyListener.notifyList;
List<PreEventContext> preNotifyList = DummyPreListener.notifyList;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), listSize);
driver.run("create database " + dbName);
listSize++;
PreCreateDatabaseEvent preDbEvent = (PreCreateDatabaseEvent) (preNotifyList.get(preNotifyList.size() - 1));
Database db = msc.getDatabase(dbName);
assertEquals(listSize, notifyList.size());
assertEquals(listSize + 1, preNotifyList.size());
validateCreateDb(db, preDbEvent.getDatabase());
CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
assert dbEvent.getStatus();
validateCreateDb(db, dbEvent.getDatabase());
driver.run("use " + dbName);
driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
PreCreateTableEvent preTblEvent = (PreCreateTableEvent) (preNotifyList.get(preNotifyList.size() - 1));
listSize++;
Table tbl = msc.getTable(dbName, tblName);
validateCreateTable(tbl, preTblEvent.getTable());
assertEquals(notifyList.size(), listSize);
CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
assert tblEvent.getStatus();
validateCreateTable(tbl, tblEvent.getTable());
driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')");
// creates index table internally
listSize += 2;
assertEquals(notifyList.size(), listSize);
AddIndexEvent addIndexEvent = (AddIndexEvent) notifyList.get(listSize - 1);
assert addIndexEvent.getStatus();
PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent) (preNotifyList.get(preNotifyList.size() - 3));
Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
validateAddIndex(oldIndex, addIndexEvent.getIndex());
validateAddIndex(oldIndex, preAddIndexEvent.getIndex());
driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + "('prop1'='val1_new', 'prop3'='val3')");
listSize++;
assertEquals(notifyList.size(), listSize);
Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i");
AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1);
assert alterIndexEvent.getStatus();
validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), newIndex, alterIndexEvent.getNewIndex());
PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), newIndex, preAlterIndexEvent.getNewIndex());
driver.run("drop index tmptbl_i on tmptbl");
listSize++;
assertEquals(notifyList.size(), listSize);
DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1);
assert dropIndexEvent.getStatus();
validateDropIndex(newIndex, dropIndexEvent.getIndex());
PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateDropIndex(newIndex, preDropIndexEvent.getIndex());
driver.run("alter table tmptbl add partition (b='2011')");
listSize++;
assertEquals(notifyList.size(), listSize);
PreAddPartitionEvent prePartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assert partEvent.getStatus();
Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
Partition partAdded = partEvent.getPartitionIterator().next();
validateAddPartition(part, partAdded);
validateTableInAddPartition(tbl, partEvent.getTable());
validateAddPartition(part, prePartEvent.getPartitions().get(0));
// Test adding multiple partitions in a single partition-set, atomically.
int currentTime = (int) System.currentTimeMillis();
HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(hiveConf);
Table table = hmsClient.getTable(dbName, "tmptbl");
Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition2 = new Partition(Arrays.asList("20110102"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition3 = new Partition(Arrays.asList("20110103"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
++listSize;
AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable());
List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterPartitionEvent preAlterPartEvent = (PreAlterPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
//the partition did not change,
// so the new partition should be similar to the original partition
Partition origP = msc.getPartition(dbName, tblName, "b=2011");
AlterPartitionEvent alterPartEvent = (AlterPartitionEvent) notifyList.get(listSize - 1);
assert alterPartEvent.getStatus();
validateAlterPartition(origP, origP, alterPartEvent.getOldPartition().getDbName(), alterPartEvent.getOldPartition().getTableName(), alterPartEvent.getOldPartition().getValues(), alterPartEvent.getNewPartition());
validateAlterPartition(origP, origP, preAlterPartEvent.getDbName(), preAlterPartEvent.getTableName(), preAlterPartEvent.getNewPartition().getValues(), preAlterPartEvent.getNewPartition());
List<String> part_vals = new ArrayList<String>();
part_vals.add("c=2012");
int preEventListSize;
preEventListSize = preNotifyList.size() + 1;
Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
listSize++;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), preEventListSize);
AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
Partition partAppended = appendPartEvent.getPartitionIterator().next();
validateAddPartition(newPart, partAppended);
PreAddPartitionEvent preAppendPartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateAddPartition(newPart, preAppendPartEvent.getPartitions().get(0));
driver.run(String.format("alter table %s rename to %s", tblName, renamed));
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterTableEvent preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
Table renamedTable = msc.getTable(dbName, renamed);
AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
assert alterTableE.getStatus();
validateAlterTable(tbl, renamedTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTable(tbl, renamedTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
//change the table name back
driver.run(String.format("alter table %s rename to %s", renamed, tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
driver.run(String.format("alter table %s ADD COLUMNS (c int)", tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
Table altTable = msc.getTable(dbName, tblName);
alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
assert alterTableE.getStatus();
validateAlterTableColumns(tbl, altTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTableColumns(tbl, altTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
Map<String, String> kvs = new HashMap<String, String>(1);
kvs.put("b", "2011");
msc.markPartitionForEvent("hive2038", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
listSize++;
assertEquals(notifyList.size(), listSize);
LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent) notifyList.get(listSize - 1);
assert partMarkEvent.getStatus();
validateLoadPartitionDone("tmptbl", kvs, partMarkEvent.getTable().getTableName(), partMarkEvent.getPartitionName());
PreLoadPartitionDoneEvent prePartMarkEvent = (PreLoadPartitionDoneEvent) preNotifyList.get(preNotifyList.size() - 1);
validateLoadPartitionDone("tmptbl", kvs, prePartMarkEvent.getTableName(), prePartMarkEvent.getPartitionName());
driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropPartitionEvent preDropPart = (PreDropPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
DropPartitionEvent dropPart = (DropPartitionEvent) notifyList.get(listSize - 1);
assert dropPart.getStatus();
validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator());
validateTableInDropPartition(tbl, dropPart.getTable());
validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator());
validateTableInDropPartition(tbl, preDropPart.getTable());
driver.run("drop table " + tblName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropTableEvent preDropTbl = (PreDropTableEvent) preNotifyList.get(preNotifyList.size() - 1);
DropTableEvent dropTbl = (DropTableEvent) notifyList.get(listSize - 1);
assert dropTbl.getStatus();
validateDropTable(tbl, dropTbl.getTable());
validateDropTable(tbl, preDropTbl.getTable());
driver.run("drop database " + dbName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropDatabaseEvent preDropDB = (PreDropDatabaseEvent) preNotifyList.get(preNotifyList.size() - 1);
DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
assert dropDB.getStatus();
validateDropDb(db, dropDB.getDatabase());
validateDropDb(db, preDropDB.getDatabase());
SetProcessor.setVariable("metaconf:hive.metastore.try.direct.sql", "false");
ConfigChangeEvent event = (ConfigChangeEvent) notifyList.get(notifyList.size() - 1);
assertEquals("hive.metastore.try.direct.sql", event.getKey());
assertEquals("true", event.getOldValue());
assertEquals("false", event.getNewValue());
}
Aggregations