Examples with MessageFormatFilter org.apache.hadoop.hive.metastore.messaging.event.filters.MessageFormatFilter used on opensource projects

Example 1 with MessageFormatFilter

use of org.apache.hadoop.hive.metastore.messaging.event.filters.MessageFormatFilter in project hive by apache.

the class TestReplicationScenarios method testEventFilters.

@Test
public void testEventFilters() {
    // Test testing that the filters introduced by EventUtils are working correctly.
    // The current filters we use in ReplicationSemanticAnalyzer is as follows:
    // IMetaStoreClient.NotificationFilter evFilter = EventUtils.andFilter(
    // EventUtils.getDbTblNotificationFilter(dbNameOrPattern, tblNameOrPattern),
    // EventUtils.getEventBoundaryFilter(eventFrom, eventTo),
    // EventUtils.restrictByMessageFormat(MessageFactory.getInstance().getMessageFormat()));
    // So, we test each of those three filters, and then test andFilter itself.
    String dbname = "testfilter_db";
    String tblname = "testfilter_tbl";
    // Test EventUtils.getDbTblNotificationFilter - this is supposed to restrict
    // events to those that match the dbname and tblname provided to the filter.
    // If the tblname passed in to the filter is null, then it restricts itself
    // to dbname-matching alone.
    IMetaStoreClient.NotificationFilter dbTblFilter = new DatabaseAndTableFilter(dbname, tblname);
    IMetaStoreClient.NotificationFilter dbFilter = new DatabaseAndTableFilter(dbname, null);
    assertFalse(dbTblFilter.accept(null));
    assertTrue(dbTblFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    assertFalse(dbFilter.accept(null));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    // Test EventUtils.getEventBoundaryFilter - this is supposed to only allow events
    // within a range specified.
    long evBegin = 50;
    long evEnd = 75;
    IMetaStoreClient.NotificationFilter evRangeFilter = new EventBoundaryFilter(evBegin, evEnd);
    assertTrue(evBegin < evEnd);
    assertFalse(evRangeFilter.accept(null));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin + 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd)));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd + 1)));
    // Test EventUtils.restrictByMessageFormat - this restricts events generated to those
    // that match a provided message format
    IMetaStoreClient.NotificationFilter restrictByDefaultMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat());
    IMetaStoreClient.NotificationFilter restrictByArbitraryMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat() + "_bogus");
    NotificationEvent dummyEvent = createDummyEvent(dbname, tblname, 0);
    assertEquals(MessageFactory.getInstance().getMessageFormat(), dummyEvent.getMessageFormat());
    assertFalse(restrictByDefaultMessageFormat.accept(null));
    assertTrue(restrictByDefaultMessageFormat.accept(dummyEvent));
    assertFalse(restrictByArbitraryMessageFormat.accept(dummyEvent));
    // Test andFilter operation.
    IMetaStoreClient.NotificationFilter yes = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return true;
        }
    };
    IMetaStoreClient.NotificationFilter no = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return false;
        }
    };
    assertTrue(new AndFilter(yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no).accept(dummyEvent));
    assertTrue(new AndFilter(yes, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, no).accept(dummyEvent));
}

Example 2 with MessageFormatFilter

use of org.apache.hadoop.hive.metastore.messaging.event.filters.MessageFormatFilter in project hive by apache.

the class ReplDumpTask method incrementalDump.

private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // get list of events matching dbPattern & tblPattern
    Long lastReplId;
    // go through each event, and dump out each event to a event-level dump dir inside dumproot
    // TODO : instead of simply restricting by message format, we should eventually
    // move to a jdbc-driver-stype registering of message format, and picking message
    // factory per event to decode. For now, however, since all messages have the
    // same factory, restricting by message format is effectively a guard against
    // older leftover data that would cause us problems.
    work.overrideEventTo(getHive());
    IMetaStoreClient.NotificationFilter evFilter = new AndFilter(new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern), new EventBoundaryFilter(work.eventFrom, work.eventTo), new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat()));
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(getHive().getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, work.eventFrom, work.maxEventLimit(), evFilter);
    lastReplId = work.eventTo;
    String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty()) ? work.dbNameOrPattern : "?";
    replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(), evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
    replLogger.startLog();
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        lastReplId = ev.getEventId();
        Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
        dumpEvent(ev, evRoot, cmRoot);
    }
    replLogger.endLog(lastReplId.toString());
    LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
    Utils.writeOutput(Arrays.asList("incremental", String.valueOf(work.eventFrom), String.valueOf(lastReplId)), dmd.getDumpFilePath(), conf);
    dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot);
    dmd.write();
    return lastReplId;
}