Examples with NotificationEvent org.apache.hadoop.hive.metastore.api.NotificationEvent used on opensource projects

Example 11 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestDbNotificationListener method sqlCTAS.

@Test
public void sqlCTAS() throws Exception {
    String sourceTblName = "sqlctasins1";
    String targetTblName = "sqlctasins2";
    // Event 1
    driver.run("create table " + sourceTblName + " (c int)");
    // Event 2 (alter: marker stats event), 3 (insert), 4 (alter: stats update event)
    driver.run("insert into table " + sourceTblName + " values (1)");
    // Event 5, 6 (alter: stats update event)
    driver.run("create table " + targetTblName + " as select c from " + sourceTblName);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(6, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(0);
    assertEquals(firstEventId + 1, event.getEventId());
    assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType());
    event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertEquals(EventType.INSERT.toString(), event.getEventType());
    // Parse the message field
    verifyInsert(event, null, sourceTblName);
    event = rsp.getEvents().get(4);
    assertEquals(firstEventId + 5, event.getEventId());
    assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType());
}

Example 12 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestDbNotificationListener method addPartition.

@Test
public void addPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "addptn";
    String tblName2 = "addptn2";
    String tblOwner = "me";
    String serdeLocation = "file:/tmp";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(1);
    assertEquals(firstEventId + 2, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    AddPartitionMessage addPtnMsg = md.getAddPartitionMessage(event.getMessage());
    assertEquals(defaultDbName, addPtnMsg.getDB());
    assertEquals(tblName, addPtnMsg.getTable());
    Iterator<Partition> ptnIter = addPtnMsg.getPartitionObjs().iterator();
    assertTrue(ptnIter.hasNext());
    assertEquals(partition, ptnIter.next());
    assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    partition = new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName2, startTime, startTime, sd, emptyParameters);
    DummyRawStoreFailEvent.setEventSucceed(false);
    try {
        msClient.add_partition(partition);
        fail("Error: add partition should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
}

Example 13 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestDbNotificationListener method dropPartition.

@Test
public void dropPartition() throws Exception {
    String defaultDbName = "default";
    String tblName = "dropptn";
    String tblOwner = "me";
    String serdeLocation = "file:/tmp";
    FieldSchema col1 = new FieldSchema("col1", "int", "no comment");
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(col1);
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, emptyParameters);
    FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment");
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    List<String> partCol1Vals = Arrays.asList("today");
    partCols.add(partCol1);
    Table table = new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Partition partition = new Partition(partCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    // Event 2
    msClient.add_partition(partition);
    // Event 3
    msClient.dropPartition(defaultDbName, tblName, partCol1Vals, false);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.DROP_PARTITION.toString(), event.getEventType());
    assertEquals(defaultDbName, event.getDbName());
    assertEquals(tblName, event.getTableName());
    // Parse the message field
    DropPartitionMessage dropPtnMsg = md.getDropPartitionMessage(event.getMessage());
    assertEquals(defaultDbName, dropPtnMsg.getDB());
    assertEquals(tblName, dropPtnMsg.getTable());
    Table tableObj = dropPtnMsg.getTableObj();
    assertEquals(table.getDbName(), tableObj.getDbName());
    assertEquals(table.getTableName(), tableObj.getTableName());
    assertEquals(table.getOwner(), tableObj.getOwner());
    assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType());
    // Verify the eventID was passed to the non-transactional listener
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2);
    MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1);
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    List<String> newpartCol1Vals = Arrays.asList("tomorrow");
    partition = new Partition(newpartCol1Vals, defaultDbName, tblName, startTime, startTime, sd, emptyParameters);
    msClient.add_partition(partition);
    DummyRawStoreFailEvent.setEventSucceed(false);
    try {
        msClient.dropPartition(defaultDbName, tblName, newpartCol1Vals, false);
        fail("Error: drop partition should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(4, rsp.getEventsSize());
}

Example 14 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestReplicationScenarios method testIncrementalLoadWithVariableLengthEventId.

@Test
public void testIncrementalLoadWithVariableLengthEventId() throws IOException, TException {
    String testName = "incrementalLoadWithVariableLengthEventId";
    String dbName = createDB(testName, driver);
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
    advanceDumpDir();
    run("REPL DUMP " + dbName, driver);
    String replDumpLocn = getResult(0, 0, driver);
    String replDumpId = getResult(0, 1, true, driver);
    LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
    run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
    // CREATE_TABLE - TRUNCATE - INSERT - The result is just one record.
    // Creating dummy table to control the event ID of TRUNCATE not to be 10 or 100 or 1000...
    String[] unptn_data = new String[] { "eleven" };
    run("CREATE TABLE " + dbName + ".dummy(a string) STORED AS TEXTFILE", driver);
    run("TRUNCATE TABLE " + dbName + ".unptned", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where all events will get ID less than 100 except TRUNCATE which will get ID 100.
    // This enesures variable length of event ID in the incremental dump
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> eventIdModifier = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        // Initialize to 0 as for increment dump, 0 won't be used.
        private long nextEventId = 0;

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventIdList) {
            if (null != eventIdList) {
                List<NotificationEvent> eventIds = eventIdList.getEvents();
                List<NotificationEvent> outEventIds = new ArrayList<NotificationEvent>();
                for (int i = 0; i < eventIds.size(); i++) {
                    NotificationEvent event = eventIds.get(i);
                    // Skip all the events belong to other DBs/tables.
                    if (event.getDbName().equalsIgnoreCase(dbName)) {
                        // CREATE_TABLE - 5, TRUNCATE - 20(20 <= Id < 100), INSERT - 100
                        switch(event.getEventType()) {
                            case "CREATE_TABLE":
                                {
                                    // The next ID is set to 20 or 200 or 2000 ... based on length of current event ID
                                    // This is done to ensure TRUNCATE doesn't get an ID 10 or 100...
                                    nextEventId = (long) Math.pow(10.0, (double) String.valueOf(event.getEventId()).length()) * 2;
                                    break;
                                }
                            case "INSERT":
                                {
                                    // INSERT will come always after CREATE_TABLE, TRUNCATE. So, no need to validate nextEventId
                                    nextEventId = (long) Math.pow(10.0, (double) String.valueOf(nextEventId).length());
                                    LOG.info("Changed EventId #{} to #{}", event.getEventId(), nextEventId);
                                    event.setEventId(nextEventId++);
                                    break;
                                }
                            default:
                                {
                                    // After CREATE_TABLE all the events in this DB should get an ID >= 20 or 200 ...
                                    if (nextEventId > 0) {
                                        LOG.info("Changed EventId #{} to #{}", event.getEventId(), nextEventId);
                                        event.setEventId(nextEventId++);
                                    }
                                    break;
                                }
                        }
                        outEventIds.add(event);
                    }
                }
                injectionPathCalled = true;
                if (outEventIds.isEmpty()) {
                    // If not even one event belongs to current DB, then return original one itself.
                    return eventIdList;
                } else {
                    // If the new list is not empty (input list have some events from this DB), then return it
                    return new NotificationEventResponse(outEventIds);
                }
            } else {
                return null;
            }
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventIdModifier);
    // It is possible that currentNotificationEventID from metastore is less than newly set event ID by stub function.
    // In this case, REPL DUMP will skip events beyond this upper limit.
    // So, to avoid this failure, we will set the TO clause to ID 100 times of currentNotificationEventID
    String cmd = "REPL DUMP " + dbName + " FROM " + replDumpId + " TO " + String.valueOf(metaStoreClient.getCurrentNotificationEventId().getEventId() * 100);
    advanceDumpDir();
    run(cmd, driver);
    eventIdModifier.assertInjectionsPerformed(true, false);
    // reset the behaviour
    InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    String incrementalDumpLocn = getResult(0, 0, driver);
    String incrementalDumpId = getResult(0, 1, true, driver);
    LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
    run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
    verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data, driverMirror);
}

Example 15 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestReplicationScenarios method testEventTypesForDynamicAddPartitionByInsert.

@Test
public void testEventTypesForDynamicAddPartitionByInsert() throws IOException {
    String name = testName.getMethodName();
    final String dbName = createDB(name, driver);
    String replDbName = dbName + "_dupe";
    run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver);
    Tuple bootstrap = bootstrapLoadAndVerify(dbName, replDbName);
    String[] ptn_data = new String[] { "ten" };
    run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data[0] + "')", driver);
    // Inject a behaviour where it throws exception if an INSERT event is found
    // As we dynamically add a partition through INSERT INTO cmd, it should just add ADD_PARTITION
    // event not an INSERT event
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> eventTypeValidator = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventsList) {
            if (null != eventsList) {
                List<NotificationEvent> events = eventsList.getEvents();
                for (int i = 0; i < events.size(); i++) {
                    NotificationEvent event = events.get(i);
                    // Skip all the events belong to other DBs/tables.
                    if (event.getDbName().equalsIgnoreCase(dbName)) {
                        if (event.getEventType().equalsIgnoreCase("INSERT")) {
                            // If an insert event is found, then return null hence no event is dumped.
                            LOG.error("Encountered INSERT event when it was not expected to");
                            return null;
                        }
                    }
                }
                injectionPathCalled = true;
            }
            return eventsList;
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventTypeValidator);
    incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName);
    eventTypeValidator.assertInjectionsPerformed(true, false);
    // reset the behaviour
    InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    verifyRun("SELECT a from " + replDbName + ".ptned where (b=1)", ptn_data, driverMirror);
}