use of org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent in project hive by apache.
the class TestMetaStoreEventListener method testListener.
@Test
public void testListener() throws Exception {
int listSize = 0;
List<ListenerEvent> notifyList = DummyListener.notifyList;
List<PreEventContext> preNotifyList = DummyPreListener.notifyList;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), listSize);
new DatabaseBuilder().setName(dbName).create(msc, conf);
listSize++;
PreCreateDatabaseEvent preDbEvent = (PreCreateDatabaseEvent) (preNotifyList.get(preNotifyList.size() - 1));
Database db = msc.getDatabase(dbName);
assertEquals(listSize, notifyList.size());
assertEquals(listSize + 1, preNotifyList.size());
validateCreateDb(db, preDbEvent.getDatabase());
CreateDatabaseEvent dbEvent = (CreateDatabaseEvent) (notifyList.get(listSize - 1));
Assert.assertTrue(dbEvent.getStatus());
validateCreateDb(db, dbEvent.getDatabase());
Table table = new TableBuilder().inDb(db).setTableName(tblName).addCol("a", "string").addPartCol("b", "string").create(msc, conf);
PreCreateTableEvent preTblEvent = (PreCreateTableEvent) (preNotifyList.get(preNotifyList.size() - 1));
listSize++;
Table tbl = msc.getTable(dbName, tblName);
validateCreateTable(tbl, preTblEvent.getTable());
assertEquals(notifyList.size(), listSize);
CreateTableEvent tblEvent = (CreateTableEvent) (notifyList.get(listSize - 1));
Assert.assertTrue(tblEvent.getStatus());
validateCreateTable(tbl, tblEvent.getTable());
new PartitionBuilder().inTable(table).addValue("2011").addToTable(msc, conf);
listSize++;
assertEquals(notifyList.size(), listSize);
PreAddPartitionEvent prePartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
AddPartitionEvent partEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
Assert.assertTrue(partEvent.getStatus());
Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
Partition partAdded = partEvent.getPartitionIterator().next();
partAdded.setWriteId(part.getWriteId());
validateAddPartition(part, partAdded);
validateTableInAddPartition(tbl, partEvent.getTable());
validateAddPartition(part, prePartEvent.getPartitions().get(0));
// Test adding multiple partitions in a single partition-set, atomically.
int currentTime = (int) System.currentTimeMillis();
HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(conf);
table = hmsClient.getTable(dbName, "tmptbl");
Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition2 = new Partition(Arrays.asList("20110102"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
Partition partition3 = new Partition(Arrays.asList("20110103"), dbName, "tmptbl", currentTime, currentTime, table.getSd(), table.getParameters());
hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
++listSize;
AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
validateTableInAddPartition(table, multiplePartitionEvent.getTable());
List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
part.setLastAccessTime((int) (System.currentTimeMillis() / 1000));
msc.alter_partition(dbName, tblName, part);
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterPartitionEvent preAlterPartEvent = (PreAlterPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
// the partition did not change,
// so the new partition should be similar to the original partition
Partition origP = msc.getPartition(dbName, tblName, "b=2011");
AlterPartitionEvent alterPartEvent = (AlterPartitionEvent) notifyList.get(listSize - 1);
Assert.assertTrue(alterPartEvent.getStatus());
validateAlterPartition(origP, origP, alterPartEvent.getOldPartition().getDbName(), alterPartEvent.getOldPartition().getTableName(), alterPartEvent.getOldPartition().getValues(), alterPartEvent.getNewPartition());
validateAlterPartition(origP, origP, preAlterPartEvent.getDbName(), preAlterPartEvent.getTableName(), preAlterPartEvent.getNewPartition().getValues(), preAlterPartEvent.getNewPartition());
List<String> part_vals = new ArrayList<>();
part_vals.add("c=2012");
int preEventListSize;
preEventListSize = preNotifyList.size() + 1;
Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
listSize++;
assertEquals(notifyList.size(), listSize);
assertEquals(preNotifyList.size(), preEventListSize);
AddPartitionEvent appendPartEvent = (AddPartitionEvent) (notifyList.get(listSize - 1));
Partition partAppended = appendPartEvent.getPartitionIterator().next();
validateAddPartition(newPart, partAppended);
PreAddPartitionEvent preAppendPartEvent = (PreAddPartitionEvent) (preNotifyList.get(preNotifyList.size() - 1));
validateAddPartition(newPart, preAppendPartEvent.getPartitions().get(0));
Table renamedTable = new Table(table);
renamedTable.setTableName(renamed);
msc.alter_table(dbName, tblName, renamedTable);
listSize++;
assertEquals(notifyList.size(), listSize);
PreAlterTableEvent preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
renamedTable = msc.getTable(dbName, renamed);
AlterTableEvent alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
Assert.assertTrue(alterTableE.getStatus());
validateAlterTable(tbl, renamedTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTable(tbl, renamedTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
// change the table name back
table = new Table(renamedTable);
table.setTableName(tblName);
msc.alter_table(dbName, renamed, table);
listSize++;
assertEquals(notifyList.size(), listSize);
table = msc.getTable(dbName, tblName);
table.getSd().addToCols(new FieldSchema("c", "int", ""));
msc.alter_table(dbName, tblName, table);
listSize++;
assertEquals(notifyList.size(), listSize);
preAlterTableE = (PreAlterTableEvent) preNotifyList.get(preNotifyList.size() - 1);
Table altTable = msc.getTable(dbName, tblName);
alterTableE = (AlterTableEvent) notifyList.get(listSize - 1);
Assert.assertTrue(alterTableE.getStatus());
validateAlterTableColumns(tbl, altTable, alterTableE.getOldTable(), alterTableE.getNewTable());
validateAlterTableColumns(tbl, altTable, preAlterTableE.getOldTable(), preAlterTableE.getNewTable());
Map<String, String> kvs = new HashMap<>(1);
kvs.put("b", "2011");
msc.markPartitionForEvent("hive2038", "tmptbl", kvs, PartitionEventType.LOAD_DONE);
listSize++;
assertEquals(notifyList.size(), listSize);
LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent) notifyList.get(listSize - 1);
Assert.assertTrue(partMarkEvent.getStatus());
validateLoadPartitionDone("tmptbl", kvs, partMarkEvent.getTable().getTableName(), partMarkEvent.getPartitionName());
PreLoadPartitionDoneEvent prePartMarkEvent = (PreLoadPartitionDoneEvent) preNotifyList.get(preNotifyList.size() - 1);
validateLoadPartitionDone("tmptbl", kvs, prePartMarkEvent.getTableName(), prePartMarkEvent.getPartitionName());
msc.dropPartition(dbName, tblName, Collections.singletonList("2011"));
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropPartitionEvent preDropPart = (PreDropPartitionEvent) preNotifyList.get(preNotifyList.size() - 1);
DropPartitionEvent dropPart = (DropPartitionEvent) notifyList.get(listSize - 1);
Assert.assertTrue(dropPart.getStatus());
validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator());
validateTableInDropPartition(tbl, dropPart.getTable());
validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator());
validateTableInDropPartition(tbl, preDropPart.getTable());
msc.dropTable(dbName, tblName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropTableEvent preDropTbl = (PreDropTableEvent) preNotifyList.get(preNotifyList.size() - 1);
DropTableEvent dropTbl = (DropTableEvent) notifyList.get(listSize - 1);
Assert.assertTrue(dropTbl.getStatus());
validateDropTable(tbl, dropTbl.getTable());
validateDropTable(tbl, preDropTbl.getTable());
msc.dropDatabase(dbName);
listSize++;
assertEquals(notifyList.size(), listSize);
PreDropDatabaseEvent preDropDB = (PreDropDatabaseEvent) preNotifyList.get(preNotifyList.size() - 1);
DropDatabaseEvent dropDB = (DropDatabaseEvent) notifyList.get(listSize - 1);
Assert.assertTrue(dropDB.getStatus());
validateDropDb(db, dropDB.getDatabase());
validateDropDb(db, preDropDB.getDatabase());
msc.setMetaConf("metastore.try.direct.sql", "false");
ConfigChangeEvent event = (ConfigChangeEvent) notifyList.get(notifyList.size() - 1);
assertEquals("metastore.try.direct.sql", event.getKey());
assertEquals("true", event.getOldValue());
assertEquals("false", event.getNewValue());
}
Aggregations