Examples with NotificationEventResponse org.apache.hadoop.hive.metastore.api.NotificationEventResponse used on opensource projects

Example 36 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method skipCleanedUpEvents.

/**
 * Test makes sure that if you use the API {@link HiveMetaStoreClient#getNextNotification(NotificationEventRequest, boolean, NotificationFilter)}
 * does not error out if the events are cleanedup.
 */
@Test
public void skipCleanedUpEvents() throws Exception {
    Database db = new Database("cleanup1", "no description", testTempDir, emptyParameters);
    msClient.createDatabase(db);
    msClient.dropDatabase("cleanup1");
    // sleep for expiry time, and then fetch again
    // sleep twice the TTL interval - things should have been cleaned by then.
    Thread.sleep(EVENTS_TTL * 2 * 1000);
    db = new Database("cleanup2", "no description", testTempDir, emptyParameters);
    msClient.createDatabase(db);
    msClient.dropDatabase("cleanup2");
    // the firstEventId is before the cleanup happened, so we should just receive the
    // events which remaining after cleanup.
    NotificationEventRequest request = new NotificationEventRequest();
    request.setLastEvent(firstEventId);
    request.setMaxEvents(-1);
    NotificationEventResponse rsp2 = msClient.getNextNotification(request, true, null);
    assertEquals(2, rsp2.getEventsSize());
    // when we pass the allowGapsInEvents as false the API should error out
    Exception ex = null;
    try {
        NotificationEventResponse rsp = msClient.getNextNotification(request, false, null);
    } catch (Exception e) {
        ex = e;
    }
    assertNotNull(ex);
}

Example 37 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method filter.

@Test
public void filter() throws Exception {
    Database db = new Database("f1", "no description", testTempDir, emptyParameters);
    msClient.createDatabase(db);
    db = new Database("f2", "no description", testTempDir, emptyParameters);
    msClient.createDatabase(db);
    msClient.dropDatabase("f2");
    IMetaStoreClient.NotificationFilter filter = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent event) {
            return event.getEventType().equals(EventType.DROP_DATABASE.toString());
        }
    };
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, filter);
    assertEquals(1, rsp.getEventsSize());
    assertEquals(firstEventId + 3, rsp.getEvents().get(0).getEventId());
}

Example 38 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method alterTable.

@Test
public void alterTable() throws Exception {
    String defaultDbName = "default";
    String tblName = "altertabletbl";
    String tblOwner = "me";
    String serdeLocation = testTempDir;
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    FieldSchema col2 = new FieldSchema("col2", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, new ArrayList<FieldSchema>(), emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    cols.add(col2);
    table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, new ArrayList<FieldSchema>(), emptyParameters, null, null, null);
    // Event 2
    msClient.alter_table(defaultDbName, tblName, table);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.ALTER_TABLE.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage());
    assertEquals(table, alterTableMessage.getTableObjAfter());
    assertEquals(TableType.MANAGED_TABLE.toString(), alterTableMessage.getTableType());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_TABLE, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    DummyRawStoreFailEvent.setEventSucceed(false);
    try {
        msClient.alter_table(defaultDbName, tblName, table);
        fail("Error: alter table should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    testEventCounts(defaultDbName, firstEventId, null, null, 2);
}

Example 39 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method openTxn.

@Test
public void openTxn() throws Exception {
    msClient.openTxn("me", TxnType.READ_ONLY);
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(0, rsp.getEventsSize());
    msClient.openTxn("me", TxnType.DEFAULT);
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(1, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(0);
    assertEquals(firstEventId + 1, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.OPEN_TXN.toString(), event.getEventType());
}

Example 40 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method sqlInsertPartition.

@Test
public void sqlInsertPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "sqlinsptn";
    // Event 1
    driver.run("create table " + tblName + " (c int) partitioned by (ds string)");
    // Event 2, 3, 4
    driver.run("insert into table " + tblName + " partition (ds = 'today') values (1)");
    // Event 5, 6, 7
    driver.run("insert into table " + tblName + " partition (ds = 'today') values (2)");
    // Event 8, 9, 10
    driver.run("insert into table " + tblName + " partition (ds) values (3, 'today')");
    // Event 9, 10
    driver.run("alter table " + tblName + " add partition (ds = 'yesterday')");
    testEventCounts(defaultDbName, firstEventId, null, null, 13);
    // Test a limit higher than available events
    testEventCounts(defaultDbName, firstEventId, null, 100, 13);
    // Test toEventId lower than current eventId
    testEventCounts(defaultDbName, firstEventId, firstEventId + 5, null, 5);
    // Event 10, 11, 12
    driver.run("insert into table " + tblName + " partition (ds = 'yesterday') values (2)");
    // Event 12, 13, 14
    driver.run("insert into table " + tblName + " partition (ds) values (3, 'yesterday')");
    // Event 15, 16, 17
    driver.run("insert into table " + tblName + " partition (ds) values (3, 'tomorrow')");
    // Event 18
    driver.run("alter table " + tblName + " drop partition (ds = 'tomorrow')");
    // Event 19, 20, 21
    driver.run("insert into table " + tblName + " partition (ds) values (42, 'todaytwo')");
    // Event 22, 23, 24
    driver.run("insert overwrite table " + tblName + " partition(ds='todaytwo') select c from " + tblName + " where 'ds'='today'");
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(31, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    event = rsp.getEvents().get(3);
    assertEquals(firstEventId + 4, event.getEventId());
    assertEquals(EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), event.getEventType());
    event = rsp.getEvents().get(4);
    assertEquals(firstEventId + 5, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, tblName);
    // Verify the replace flag.
    InsertMessage insertMsg = md.getInsertMessage(event.getMessage());
    assertFalse(insertMsg.isReplace());
    event = rsp.getEvents().get(8);
    assertEquals(firstEventId + 9, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, tblName);
    event = rsp.getEvents().get(12);
    assertEquals(firstEventId + 13, event.getEventId());
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    event = rsp.getEvents().get(13);
    assertEquals(firstEventId + 14, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, tblName);
    event = rsp.getEvents().get(17);
    assertEquals(firstEventId + 18, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, tblName);
    event = rsp.getEvents().get(21);
    assertEquals(firstEventId + 22, event.getEventId());
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    event = rsp.getEvents().get(24);
    assertEquals(firstEventId + 25, event.getEventId());
    assertEquals(EventType.DROP_PARTITION.toString(), event.getEventType());
    event = rsp.getEvents().get(25);
    assertEquals(firstEventId + 26, event.getEventId());
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    event = rsp.getEvents().get(26);
    assertEquals(firstEventId + 27, event.getEventId());
    assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType());
    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
    // Test fromEventId different from the very first
    testEventCounts(defaultDbName, event.getEventId(), null, null, 4);
    event = rsp.getEvents().get(28);
    assertEquals(firstEventId + 29, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Verify the replace flag.
    insertMsg = md.getInsertMessage(event.getMessage());
    assertTrue(insertMsg.isReplace());
    // replace-overwrite introduces no new files
    // the insert overwrite creates an empty file with the current change
    // assertTrue(event.getMessage().matches(".*\"files\":\\[\\].*"));
    event = rsp.getEvents().get(29);
    assertEquals(firstEventId + 30, event.getEventId());
    assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType());
    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
    event = rsp.getEvents().get(30);
    assertEquals(firstEventId + 31, event.getEventId());
    assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType());
    assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*"));
    testEventCounts(defaultDbName, firstEventId, null, null, 31);
    // Test a limit within the available events
    testEventCounts(defaultDbName, firstEventId, null, 10, 10);
    // Test toEventId greater than current eventId
    testEventCounts(defaultDbName, firstEventId, firstEventId + 100, null, 31);
    // Test toEventId greater than current eventId with some limit within available events
    testEventCounts(defaultDbName, firstEventId, firstEventId + 100, 10, 10);
    // Test toEventId greater than current eventId with some limit beyond available events
    testEventCounts(defaultDbName, firstEventId, firstEventId + 100, 50, 31);
}