Examples with NotificationEventResponse org.apache.hadoop.hive.metastore.api.NotificationEventResponse used on opensource projects

Example 21 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestDbNotificationListener method createIndex.

@Test
public void createIndex() throws Exception {
    String indexName = "createIndex";
    String dbName = "default";
    String tableName = "createIndexTable";
    String indexTableName = tableName + "__" + indexName + "__";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", ""));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    Map<String, String> params = new HashMap<String, String>();
    params.put("key", "value");
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 1
    msClient.createTable(table);
    Index index = new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, emptyParameters, false);
    Table indexTable = new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    // Event 2, 3 (index table and index)
    msClient.createIndex(index, indexTable);
    // Get notifications from metastore
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
    NotificationEvent event = rsp.getEvents().get(2);
    assertEquals(firstEventId + 3, event.getEventId());
    assertTrue(event.getEventTime() >= startTime);
    assertEquals(EventType.CREATE_INDEX.toString(), event.getEventType());
    assertEquals(dbName, event.getDbName());
    // Parse the message field
    CreateIndexMessage createIdxMessage = md.getCreateIndexMessage(event.getMessage());
    assertEquals(dbName, createIdxMessage.getDB());
    Index indexObj = createIdxMessage.getIndexObj();
    assertEquals(dbName, indexObj.getDbName());
    assertEquals(indexName, indexObj.getIndexName());
    assertEquals(tableName, indexObj.getOrigTableName());
    assertEquals(indexTableName, indexObj.getIndexTableName());
    // When hive.metastore.transactional.event.listeners is set,
    // a failed event should not create a new notification
    DummyRawStoreFailEvent.setEventSucceed(false);
    index = new Index("createIndexTable2", null, "default", tableName, startTime, startTime, "createIndexTable2__createIndexTable2__", sd, emptyParameters, false);
    Table indexTable2 = new Table("createIndexTable2__createIndexTable2__", dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    try {
        msClient.createIndex(index, indexTable2);
        fail("Error: create index should've failed");
    } catch (Exception ex) {
    // expected
    }
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(3, rsp.getEventsSize());
}

Example 22 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestReplicationScenarios method testIdempotentMoveTaskForInsertFiles.

@Test
public void testIdempotentMoveTaskForInsertFiles() throws IOException {
    String name = testName.getMethodName();
    final String dbName = createDB(name, driver);
    String replDbName = dbName + "_dupe";
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    Tuple bootstrap = bootstrapLoadAndVerify(dbName, replDbName);
    String[] unptn_data = new String[] { "ten" };
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where it repeats the INSERT event twice with different event IDs
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> insertEventRepeater = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventsList) {
            if (null != eventsList) {
                List<NotificationEvent> events = eventsList.getEvents();
                List<NotificationEvent> outEvents = new ArrayList<>();
                long insertEventId = -1;
                for (int i = 0; i < events.size(); i++) {
                    NotificationEvent event = events.get(i);
                    // Skip all the events belong to other DBs/tables.
                    if (event.getDbName().equalsIgnoreCase(dbName)) {
                        if (event.getEventType().equalsIgnoreCase("INSERT")) {
                            // Add insert event twice with different event ID to allow apply of both events.
                            NotificationEvent newEvent = new NotificationEvent(event);
                            outEvents.add(newEvent);
                            insertEventId = newEvent.getEventId();
                        }
                    }
                    NotificationEvent newEvent = new NotificationEvent(event);
                    if (insertEventId != -1) {
                        insertEventId++;
                        newEvent.setEventId(insertEventId);
                    }
                    outEvents.add(newEvent);
                }
                eventsList.setEvents(outEvents);
                injectionPathCalled = true;
            }
            return eventsList;
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(insertEventRepeater);
    try {
        incrementalLoadAndVerify(dbName, replDbName);
        insertEventRepeater.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    }
    verifyRun("SELECT a from " + replDbName + ".unptned", unptn_data[0], driverMirror);
}

Example 23 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestReplicationScenarios method testIncrementalReplWithEventsMissing.

@Test
public void testIncrementalReplWithEventsMissing() throws IOException, TException {
    String testName = "incrementalReplWithEventsMissing";
    String dbName = createDB(testName, driver);
    String replDbName = dbName + "_dupe";
    Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
    String replDumpId = bootstrapDump.lastReplId;
    // CREATE_TABLE - INSERT - TRUNCATE - INSERT - The result is just one record.
    String[] unptn_data = new String[] { "eleven" };
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver);
    run("TRUNCATE TABLE " + dbName + ".unptned", driver);
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where some events missing from notification_log table.
    // This ensures the incremental dump doesn't get all events for replication.
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> eventIdSkipper = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventIdList) {
            if (null != eventIdList) {
                List<NotificationEvent> eventIds = eventIdList.getEvents();
                List<NotificationEvent> outEventIds = new ArrayList<NotificationEvent>();
                for (int i = 0; i < eventIds.size(); i++) {
                    NotificationEvent event = eventIds.get(i);
                    // Skip all the INSERT events
                    if (event.getDbName().equalsIgnoreCase(dbName) && event.getEventType().equalsIgnoreCase("INSERT")) {
                        injectionPathCalled = true;
                        continue;
                    }
                    outEventIds.add(event);
                }
                // Return the new list
                return new NotificationEventResponse(outEventIds);
            } else {
                return null;
            }
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventIdSkipper);
    try {
        advanceDumpDir();
        try {
            driver.run("REPL DUMP " + dbName);
            assert false;
        } catch (CommandProcessorException e) {
            assertTrue(e.getCauseMessage() == ErrorMsg.REPL_EVENTS_MISSING_IN_METASTORE.getMsg());
        }
        eventIdSkipper.assertInjectionsPerformed(true, false);
    } finally {
        // reset the behaviour
        InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    }
}

Example 24 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestReplicationScenarios method testAuthForNotificationAPIs.

@Test
public void testAuthForNotificationAPIs() throws Exception {
    // Setup
    long firstEventId = metaStoreClient.getCurrentNotificationEventId().getEventId();
    String dbName = "testAuthForNotificationAPIs";
    createDB(dbName, driver);
    NotificationEventResponse rsp = metaStoreClient.getNextNotification(firstEventId, 0, null);
    assertEquals(1, rsp.getEventsSize());
    // Test various scenarios
    // Remove the proxy privilege and the auth should fail (in reality the proxy setting should not be changed on the fly)
    hconf.unset(proxySettingName);
    // Need to explicitly update ProxyUsers
    ProxyUsers.refreshSuperUserGroupsConfiguration(hconf);
    // Verify if the auth should fail
    Exception ex = null;
    try {
        rsp = metaStoreClient.getNextNotification(firstEventId, 0, null);
    } catch (TException e) {
        ex = e;
    }
    assertNotNull(ex);
    // Disable auth so the call should succeed
    MetastoreConf.setBoolVar(hconf, MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH, false);
    try {
        rsp = metaStoreClient.getNextNotification(firstEventId, 0, null);
        assertEquals(1, rsp.getEventsSize());
    } finally {
        // Restore the settings
        MetastoreConf.setBoolVar(hconf, MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH, true);
        hconf.set(proxySettingName, "*");
        ProxyUsers.refreshSuperUserGroupsConfiguration(hconf);
    }
}

Example 25 with NotificationEventResponse

use of org.apache.hadoop.hive.metastore.api.NotificationEventResponse in project hive by apache.

the class TestReplicationScenarios method testCleanerThreadStartupWait.

@Test
public void testCleanerThreadStartupWait() throws Exception {
    int eventsTtl = 20;
    HiveConf newConf = new HiveConf(hconf);
    // Set TTL short enough for testing.
    MetastoreConf.setTimeVar(newConf, REPL_EVENT_DB_LISTENER_TTL, eventsTtl, TimeUnit.SECONDS);
    // Set startup wait interval.
    MetastoreConf.setTimeVar(newConf, EVENT_DB_LISTENER_CLEAN_STARTUP_WAIT_INTERVAL, eventsTtl * 5, TimeUnit.SECONDS);
    // Set cleaner wait interval.
    MetastoreConf.setTimeVar(newConf, MetastoreConf.ConfVars.EVENT_DB_LISTENER_CLEAN_INTERVAL, 10, TimeUnit.MILLISECONDS);
    newConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST_REPL, true);
    // Reset Cleaner to have a initial wait time.
    DbNotificationListener.resetCleaner(newConf);
    IMetaStoreClient msClient = metaStoreClient;
    run("create database referenceDb", driver);
    long firstEventId = msClient.getCurrentNotificationEventId().getEventId();
    ;
    run("create database cleanupStartup", driver);
    run("drop database cleanupStartup", driver);
    LOG.info("Done with creating events.");
    // Check events are pushed into notification logs.
    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    // Reset Cleaner to have a initial wait time.
    DbNotificationListener.resetCleaner(newConf);
    // Sleep for eventsTtl time and see if events are there.
    Thread.sleep(eventsTtl * 1000);
    // Check events are there in notification logs.
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    // Sleep for some more time and see if events are there.
    Thread.sleep(eventsTtl * 1000);
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(2, rsp.getEventsSize());
    // Sleep more than the initial wait time and see if Events get cleaned up post that
    Thread.sleep(eventsTtl * 4000);
    // Events should have cleaned up.
    rsp = msClient.getNextNotification(firstEventId, 0, null);
    assertEquals(0, rsp.getEventsSize());
    // Reset with original configuration.
    DbNotificationListener.resetCleaner(hconf);
    run("drop database referenceDb", driver);
}