Examples with NotificationEvent org.apache.hadoop.hive.metastore.api.NotificationEvent used on opensource projects

Example 26 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestObjectStore method testConcurrentAddNotifications.

@Ignore("This test is here to allow testing with other databases like mysql / postgres etc\n" + " with  user changes to the code. This cannot be run on apache derby because of\n" + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html")
@Test
public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException {
    final int NUM_THREADS = 10;
    CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS, () -> LoggerFactory.getLogger("test").debug(NUM_THREADS + " threads going to add notification"));
    Configuration conf = MetastoreConf.newMetastoreConf();
    MetaStoreTestUtils.setConfForStandloneMode(conf);
    /*
       Below are the properties that need to be set based on what database this test is going to be run
     */
    // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver");
    // conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
    // "jdbc:mysql://localhost:3306/metastore_db");
    // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "");
    // conf.setVar(HiveConf.ConfVars.METASTOREPWD, "");
    /*
     we have to  add this one manually as for tests the db is initialized via the metastoreDiretSQL
     and we don't run the schema creation sql that includes the an insert for notification_sequence
     which can be locked. the entry in notification_sequence happens via notification_event insertion.
    */
    objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute();
    objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute();
    objectStore.addNotificationEvent(new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "CREATE DATABASE DB initial"));
    ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS);
    for (int i = 0; i < NUM_THREADS; i++) {
        final int n = i;
        executorService.execute(() -> {
            ObjectStore store = new ObjectStore();
            store.setConf(conf);
            String eventType = EventMessage.EventType.CREATE_DATABASE.toString();
            NotificationEvent dbEvent = new NotificationEvent(0, 0, eventType, "CREATE DATABASE DB" + n);
            System.out.println("ADDING NOTIFICATION");
            try {
                cyclicBarrier.await();
            } catch (InterruptedException | BrokenBarrierException e) {
                throw new RuntimeException(e);
            }
            store.addNotificationEvent(dbEvent);
            System.out.println("FINISH NOTIFICATION");
        });
    }
    executorService.shutdown();
    Assert.assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS));
    // we have to setup this again as the underlying PMF keeps getting reinitialized with original
    // reference closed
    ObjectStore store = new ObjectStore();
    store.setConf(conf);
    NotificationEventResponse eventResponse = store.getNextNotification(new NotificationEventRequest());
    Assert.assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize());
    long previousId = 0;
    for (NotificationEvent event : eventResponse.getEvents()) {
        Assert.assertTrue("previous:" + previousId + " current:" + event.getEventId(), previousId < event.getEventId());
        Assert.assertTrue(previousId + 1 == event.getEventId());
        previousId = event.getEventId();
    }
}

Example 27 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestDbNotificationListener method createIndex.

@Test
public void createIndex() throws Exception {
    String indexName = "createIndex";
    String dbName = "default";
    String tableName = "createIndexTable";
    String indexTableName = tableName + "__" + indexName + "__";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", ""));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    Map<String, String> params = new HashMap<String, String>();
    params.put("key", "value");
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Index index = new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, emptyParameters, false);
    Table indexTable = new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 2, 3 (index table and index)
    msClient.createIndex(index, indexTable);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.CREATE_INDEX.toString(), event.getEventType());
    assertEquals(dbName, event.getDbName());
    // Parse the message field
    CreateIndexMessage createIdxMessage = md.getCreateIndexMessage(event.getMessage());
    assertEquals(dbName, createIdxMessage.getDB());
    Index indexObj = createIdxMessage.getIndexObj();
    assertEquals(dbName, indexObj.getDbName());
    assertEquals(indexName, indexObj.getIndexName());
    assertEquals(tableName, indexObj.getOrigTableName());
    assertEquals(indexTableName, indexObj.getIndexTableName());
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    DummyRawStoreFailEvent.setEventSucceed(false);
    index = new Index("createIndexTable2", null, "default", tableName, startTime, startTime, "createIndexTable2__createIndexTable2__", sd, emptyParameters, false);
    Table indexTable2 = new Table("createIndexTable2__createIndexTable2__", dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    try {
        msClient.createIndex(index, indexTable2);
        fail("Error: create index should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
}

Example 28 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAddIndex.

/**
   * @param indexEvent index event
   * @throws MetaException
   */
@Override
public void onAddIndex(AddIndexEvent indexEvent) throws MetaException {
    Index index = indexEvent.getIndex();
    NotificationEvent event = new NotificationEvent(0, now(), EventType.CREATE_INDEX.toString(), msgFactory.buildCreateIndexMessage(index).toString());
    event.setDbName(index.getDbName());
    process(event);
}

Example 29 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAlterIndex.

/**
   * @param indexEvent index event
   * @throws MetaException
   */
@Override
public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException {
    Index before = indexEvent.getOldIndex();
    Index after = indexEvent.getNewIndex();
    NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_INDEX.toString(), msgFactory.buildAlterIndexMessage(before, after).toString());
    event.setDbName(before.getDbName());
    process(event);
}

Example 30 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onDropIndex.

/**
   * @param indexEvent index event
   * @throws MetaException
   */
@Override
public void onDropIndex(DropIndexEvent indexEvent) throws MetaException {
    Index index = indexEvent.getIndex();
    NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_INDEX.toString(), msgFactory.buildDropIndexMessage(index).toString());
    event.setDbName(index.getDbName());
    process(event);
}