Examples with NotificationEvent org.apache.hadoop.hive.metastore.api.NotificationEvent used on opensource projects

Example 51 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestReplicationScenarios method testIdempotentMoveTaskForInsertFiles.

@Test
public void testIdempotentMoveTaskForInsertFiles() throws IOException {
    String name = testName.getMethodName();
    final String dbName = createDB(name, driver);
    String replDbName = dbName + "_dupe";
    run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver);
    Tuple bootstrap = bootstrapLoadAndVerify(dbName, replDbName);
    String[] unptn_data = new String[] { "ten" };
    run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver);
    // Inject a behaviour where it repeats the INSERT event twice with different event IDs
    BehaviourInjection<NotificationEventResponse, NotificationEventResponse> insertEventRepeater = new BehaviourInjection<NotificationEventResponse, NotificationEventResponse>() {

        @Nullable
        @Override
        public NotificationEventResponse apply(@Nullable NotificationEventResponse eventsList) {
            if (null != eventsList) {
                List<NotificationEvent> events = eventsList.getEvents();
                List<NotificationEvent> outEvents = new ArrayList<>();
                long insertEventId = -1;
                for (int i = 0; i < events.size(); i++) {
                    NotificationEvent event = events.get(i);
                    // Skip all the events belong to other DBs/tables.
                    if (event.getDbName().equalsIgnoreCase(dbName)) {
                        if (event.getEventType().equalsIgnoreCase("INSERT")) {
                            // Add insert event twice with different event ID to allow apply of both events.
                            NotificationEvent newEvent = new NotificationEvent(event);
                            outEvents.add(newEvent);
                            insertEventId = newEvent.getEventId();
                        }
                    }
                    NotificationEvent newEvent = new NotificationEvent(event);
                    if (insertEventId != -1) {
                        insertEventId++;
                        newEvent.setEventId(insertEventId);
                    }
                    outEvents.add(newEvent);
                }
                eventsList.setEvents(outEvents);
                injectionPathCalled = true;
            }
            return eventsList;
        }
    };
    InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(insertEventRepeater);
    incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName);
    insertEventRepeater.assertInjectionsPerformed(true, false);
    // reset the behaviour
    InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour();
    verifyRun("SELECT a from " + replDbName + ".unptned", unptn_data, driverMirror);
}

Example 52 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestReplicationScenarios method createDummyEvent.

private NotificationEvent createDummyEvent(String dbname, String tblname, long evid) {
    MessageFactory msgFactory = MessageFactory.getInstance();
    Table t = new Table();
    t.setDbName(dbname);
    t.setTableName(tblname);
    NotificationEvent event = new NotificationEvent(evid, (int) System.currentTimeMillis(), MessageFactory.CREATE_TABLE_EVENT, msgFactory.buildCreateTableMessage(t, Arrays.asList("/tmp/").iterator()).toString());
    event.setDbName(t.getDbName());
    event.setTableName(t.getTableName());
    event.setMessageFormat(msgFactory.getMessageFormat());
    return event;
}

Example 53 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class TestReplicationScenarios method testEventFilters.

@Test
public void testEventFilters() {
    // Test testing that the filters introduced by EventUtils are working correctly.
    // The current filters we use in ReplicationSemanticAnalyzer is as follows:
    // IMetaStoreClient.NotificationFilter evFilter = EventUtils.andFilter(
    // EventUtils.getDbTblNotificationFilter(dbNameOrPattern, tblNameOrPattern),
    // EventUtils.getEventBoundaryFilter(eventFrom, eventTo),
    // EventUtils.restrictByMessageFormat(MessageFactory.getInstance().getMessageFormat()));
    // So, we test each of those three filters, and then test andFilter itself.
    String dbname = "testfilter_db";
    String tblname = "testfilter_tbl";
    // Test EventUtils.getDbTblNotificationFilter - this is supposed to restrict
    // events to those that match the dbname and tblname provided to the filter.
    // If the tblname passed in to the filter is null, then it restricts itself
    // to dbname-matching alone.
    IMetaStoreClient.NotificationFilter dbTblFilter = new DatabaseAndTableFilter(dbname, tblname);
    IMetaStoreClient.NotificationFilter dbFilter = new DatabaseAndTableFilter(dbname, null);
    assertFalse(dbTblFilter.accept(null));
    assertTrue(dbTblFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    assertFalse(dbFilter.accept(null));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    // Test EventUtils.getEventBoundaryFilter - this is supposed to only allow events
    // within a range specified.
    long evBegin = 50;
    long evEnd = 75;
    IMetaStoreClient.NotificationFilter evRangeFilter = new EventBoundaryFilter(evBegin, evEnd);
    assertTrue(evBegin < evEnd);
    assertFalse(evRangeFilter.accept(null));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin + 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd)));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd + 1)));
    // Test EventUtils.restrictByMessageFormat - this restricts events generated to those
    // that match a provided message format
    IMetaStoreClient.NotificationFilter restrictByDefaultMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat());
    IMetaStoreClient.NotificationFilter restrictByArbitraryMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat() + "_bogus");
    NotificationEvent dummyEvent = createDummyEvent(dbname, tblname, 0);
    assertEquals(MessageFactory.getInstance().getMessageFormat(), dummyEvent.getMessageFormat());
    assertFalse(restrictByDefaultMessageFormat.accept(null));
    assertTrue(restrictByDefaultMessageFormat.accept(dummyEvent));
    assertFalse(restrictByArbitraryMessageFormat.accept(dummyEvent));
    // Test andFilter operation.
    IMetaStoreClient.NotificationFilter yes = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return true;
        }
    };
    IMetaStoreClient.NotificationFilter no = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return false;
        }
    };
    assertTrue(new AndFilter(yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no).accept(dummyEvent));
    assertTrue(new AndFilter(yes, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, no).accept(dummyEvent));
}

Example 54 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class ReplicationV1CompatRule method doBackwardCompatibilityCheck.

public void doBackwardCompatibilityCheck(long testEventIdBefore, long testEventIdAfter) {
    // try to instantiate the old replv1 task generation on every event produced.
    long timeBefore = System.currentTimeMillis();
    Map<NotificationEvent, RuntimeException> unhandledTasks = new LinkedHashMap<>();
    Map<NotificationEvent, RuntimeException> incompatibleTasks = new LinkedHashMap<>();
    int eventCount = 0;
    LOG.info("Checking replv1 backward compatibility for events between : " + testEventIdBefore + " -> " + testEventIdAfter);
    IMetaStoreClient.NotificationFilter evFilter = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return true;
        }
    };
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(metaStoreClient);
    try {
        EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, testEventIdBefore, Ints.checkedCast(testEventIdAfter - testEventIdBefore) + 1, evFilter);
        ReplicationTask.resetFactory(null);
        assertTrue("We should have found some events", evIter.hasNext());
        while (evIter.hasNext()) {
            eventCount++;
            NotificationEvent ev = evIter.next();
            // convert to HCatNotificationEvent, and then try to instantiate a ReplicationTask on it.
            try {
                ReplicationTask rtask = ReplicationTask.create(HCatClient.create(hconf), new HCatNotificationEvent(ev));
                if (rtask instanceof ErroredReplicationTask) {
                    unhandledTasks.put(ev, ((ErroredReplicationTask) rtask).getCause());
                }
            } catch (RuntimeException re) {
                incompatibleTasks.put(ev, re);
            }
        }
    } catch (IOException e) {
        assertNull("Got an exception when we shouldn't have - replv1 backward incompatibility issue:", e);
    }
    if (unhandledTasks.size() > 0) {
        LOG.warn("Events found that would not be coverable by replv1 replication: " + unhandledTasks.size());
        for (NotificationEvent ev : unhandledTasks.keySet()) {
            RuntimeException re = unhandledTasks.get(ev);
            LOG.warn("ErroredReplicationTask encountered - new event type does not correspond to a replv1 task:" + ev.toString(), re);
        }
    }
    if (incompatibleTasks.size() > 0) {
        LOG.warn("Events found that caused errors in replv1 replication: " + incompatibleTasks.size());
        for (NotificationEvent ev : incompatibleTasks.keySet()) {
            RuntimeException re = incompatibleTasks.get(ev);
            LOG.warn("RuntimeException encountered - new event type caused a replv1 break." + ev.toString(), re);
        }
    }
    assertEquals(0, incompatibleTasks.size());
    long timeAfter = System.currentTimeMillis();
    LOG.info("Backward compatibility check timing:" + timeBefore + " -> " + timeAfter + ", ev: " + testEventIdBefore + " => " + testEventIdAfter + ", #events processed=" + eventCount);
}

Example 55 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAlterDatabase.

/**
 * @param dbEvent alter database event
 * @throws MetaException
 */
@Override
public void onAlterDatabase(AlterDatabaseEvent dbEvent) throws MetaException {
    Database oldDb = dbEvent.getOldDatabase();
    Database newDb = dbEvent.getNewDatabase();
    NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_DATABASE.toString(), msgFactory.buildAlterDatabaseMessage(oldDb, newDb).toString());
    event.setDbName(oldDb.getName());
    process(event, dbEvent);
}