Examples with NotificationEventResponse org.apache.hadoop.hive.metastore.api.NotificationEventResponse used on opensource projects

Example 16 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method cleanupNotificationWithError.

@Test
public void cleanupNotificationWithError() throws Exception {
    Database db = new Database("cleanup1", "no description", testTempDir, emptyParameters);
    msClient.createDatabase(db);
    msClient.dropDatabase("cleanup1");
    LOG.info("Pulling events immediately after createDatabase/dropDatabase");
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    // this simulates that cleaning thread will error out while cleaning the notifications
    DummyRawStoreFailEvent.setEventSucceed(false);
    // sleep for expiry time, and then fetch again
    // sleep twice the TTL interval - things should have been cleaned by then.
    Thread.sleep(EVENTS_TTL * 2 * 1000);
    LOG.info("Pulling events again after failing to cleanup");
    NotificationEventResponse rsp2 = msClient.getNextNotification(firstEventId, 0, null);
    LOG.info("second trigger done");
    assertEquals(2, rsp2.getEventsSize());
    DummyRawStoreFailEvent.setEventSucceed(true);
    Thread.sleep(EVENTS_TTL * 2 * 1000);
    LOG.info("Pulling events again after cleanup");
    rsp2 = msClient.getNextNotification(firstEventId, 0, null);
    LOG.info("third trigger done");
    assertEquals(0, rsp2.getEventsSize());
}

Example 17 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method sqlCTAS.

@Test
public void sqlCTAS() throws Exception {
    String defaultDbName = "default";
    String sourceTblName = "sqlctasins1";
    String targetTblName = "sqlctasins2";
    // Event 1
    driver.run("create table " + sourceTblName + " (c int)");
    // Event 2 (alter: marker stats event), 3 (insert), 4 (alter: stats update event)
    driver.run("insert into table " + sourceTblName + " values (1)");
    // Event 5, 6 (alter), 7 (alter: stats update event)
    driver.run("create table " + targetTblName + " as select c from " + sourceTblName);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(8, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(0);
    assertEquals(firstEventId + 1, event.getEventId());
    assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType());
    event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, sourceTblName);
    event = rsp.getEvents().get(5);
    assertEquals(firstEventId + 6, event.getEventId());
    assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType());
    testEventCounts(defaultDbName, firstEventId, null, null, 8);
}

Example 18 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestObjectStore method testMaxEventResponse.

/**
 * Test metastore configuration property METASTORE_MAX_EVENT_RESPONSE
 */
@Test
public void testMaxEventResponse() throws InterruptedException, MetaException {
    NotificationEvent event = new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "");
    MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.METASTORE_MAX_EVENT_RESPONSE, 1);
    ObjectStore objs = new ObjectStore();
    objs.setConf(conf);
    // Verify if METASTORE_MAX_EVENT_RESPONSE will limit number of events to respond
    for (int i = 0; i < 3; i++) {
        objs.addNotificationEvent(event);
    }
    NotificationEventResponse eventResponse = objs.getNextNotification(new NotificationEventRequest());
    Assert.assertEquals(1, eventResponse.getEventsSize());
}

Example 19 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestObjectStore method testConcurrentAddNotifications.

@Ignore("This test is here to allow testing with other databases like mysql / postgres etc\n" + " with  user changes to the code. This cannot be run on apache derby because of\n" + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html")
@Test
public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException, MetaException {
    final int NUM_THREADS = 10;
    CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS, () -> LoggerFactory.getLogger("test").debug(NUM_THREADS + " threads going to add notification"));
    Configuration conf = MetastoreConf.newMetastoreConf();
    MetaStoreTestUtils.setConfForStandloneMode(conf);
    /*
       Below are the properties that need to be set based on what database this test is going to be run
     */
    // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver");
    // conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
    // "jdbc:mysql://localhost:3306/metastore_db");
    // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "");
    // conf.setVar(HiveConf.ConfVars.METASTOREPWD, "");
    /*
     we have to  add this one manually as for tests the db is initialized via the metastoreDiretSQL
     and we don't run the schema creation sql that includes the an insert for notification_sequence
     which can be locked. the entry in notification_sequence happens via notification_event insertion.
    */
    objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute();
    objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute();
    objectStore.addNotificationEvent(new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "CREATE DATABASE DB initial"));
    ExecutorService executorService = newFixedThreadPool(NUM_THREADS);
    for (int i = 0; i < NUM_THREADS; i++) {
        final int n = i;
        executorService.execute(() -> {
            ObjectStore store = new ObjectStore();
            store.setConf(conf);
            String eventType = EventMessage.EventType.CREATE_DATABASE.toString();
            NotificationEvent dbEvent = new NotificationEvent(0, 0, eventType, "CREATE DATABASE DB" + n);
            System.out.println("ADDING NOTIFICATION");
            try {
                cyclicBarrier.await();
                store.addNotificationEvent(dbEvent);
            } catch (InterruptedException | BrokenBarrierException | MetaException e) {
                throw new RuntimeException(e);
            }
            System.out.println("FINISH NOTIFICATION");
        });
    }
    executorService.shutdown();
    Assert.assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS));
    // we have to setup this again as the underlying PMF keeps getting reinitialized with original
    // reference closed
    ObjectStore store = new ObjectStore();
    store.setConf(conf);
    NotificationEventResponse eventResponse = store.getNextNotification(new NotificationEventRequest());
    Assert.assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize());
    long previousId = 0;
    for (NotificationEvent event : eventResponse.getEvents()) {
        Assert.assertTrue("previous:" + previousId + " current:" + event.getEventId(), previousId < event.getEventId());
        Assert.assertTrue(previousId + 1 == event.getEventId());
        previousId = event.getEventId();
    }
}

Example 20 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestObjectStore method testNotificationOps.

/**
 * Test notification operations
 */
// TODO MS-SPLIT uncomment once we move EventMessage over
@Test
public void testNotificationOps() throws InterruptedException, MetaException {
    final int NO_EVENT_ID = 0;
    final int FIRST_EVENT_ID = 1;
    final int SECOND_EVENT_ID = 2;
    NotificationEvent event = new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "");
    NotificationEventResponse eventResponse;
    CurrentNotificationEventId eventId;
    // Verify that there is no notifications available yet
    eventId = objectStore.getCurrentNotificationEventId();
    Assert.assertEquals(NO_EVENT_ID, eventId.getEventId());
    // Verify that addNotificationEvent() updates the NotificationEvent with the new event ID
    objectStore.addNotificationEvent(event);
    Assert.assertEquals(FIRST_EVENT_ID, event.getEventId());
    objectStore.addNotificationEvent(event);
    Assert.assertEquals(SECOND_EVENT_ID, event.getEventId());
    // Verify that objectStore fetches the latest notification event ID
    eventId = objectStore.getCurrentNotificationEventId();
    Assert.assertEquals(SECOND_EVENT_ID, eventId.getEventId());
    // Verify that getNextNotification() returns all events
    eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
    Assert.assertEquals(2, eventResponse.getEventsSize());
    Assert.assertEquals(FIRST_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
    Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(1).getEventId());
    // Verify that getNextNotification(last) returns events after a specified event
    eventResponse = objectStore.getNextNotification(new NotificationEventRequest(FIRST_EVENT_ID));
    Assert.assertEquals(1, eventResponse.getEventsSize());
    Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
    // Verify that getNextNotification(last) returns zero events if there are no more notifications available
    eventResponse = objectStore.getNextNotification(new NotificationEventRequest(SECOND_EVENT_ID));
    Assert.assertEquals(0, eventResponse.getEventsSize());
    // Verify that cleanNotificationEvents() cleans up all old notifications
    objectStore.cleanNotificationEvents(0);
    eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
    Assert.assertEquals(0, eventResponse.getEventsSize());
}