Examples with DatabaseAndTableFilter org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter used on opensource projects

Example 1 with DatabaseAndTableFilter

use of org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter in project hive by apache.

the class TestReplicationScenarios method testEventFilters.

@Test
public void testEventFilters() {
    // Test testing that the filters introduced by EventUtils are working correctly.
    // The current filters we use in ReplicationSemanticAnalyzer is as follows:
    // IMetaStoreClient.NotificationFilter evFilter = EventUtils.andFilter(
    // EventUtils.getDbTblNotificationFilter(dbNameOrPattern, tblNameOrPattern),
    // EventUtils.getEventBoundaryFilter(eventFrom, eventTo),
    // EventUtils.restrictByMessageFormat(MessageFactory.getInstance().getMessageFormat()));
    // So, we test each of those three filters, and then test andFilter itself.
    String dbname = "testfilter_db";
    String tblname = "testfilter_tbl";
    // Test EventUtils.getDbTblNotificationFilter - this is supposed to restrict
    // events to those that match the dbname and tblname provided to the filter.
    // If the tblname passed in to the filter is null, then it restricts itself
    // to dbname-matching alone.
    IMetaStoreClient.NotificationFilter dbTblFilter = new DatabaseAndTableFilter(dbname, tblname);
    IMetaStoreClient.NotificationFilter dbFilter = new DatabaseAndTableFilter(dbname, null);
    assertFalse(dbTblFilter.accept(null));
    assertTrue(dbTblFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbTblFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    assertFalse(dbFilter.accept(null));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname, 0)));
    assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
    assertFalse(dbFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
    // Test EventUtils.getEventBoundaryFilter - this is supposed to only allow events
    // within a range specified.
    long evBegin = 50;
    long evEnd = 75;
    IMetaStoreClient.NotificationFilter evRangeFilter = new EventBoundaryFilter(evBegin, evEnd);
    assertTrue(evBegin < evEnd);
    assertFalse(evRangeFilter.accept(null));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin + 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd - 1)));
    assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd)));
    assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd + 1)));
    // Test EventUtils.restrictByMessageFormat - this restricts events generated to those
    // that match a provided message format
    IMetaStoreClient.NotificationFilter restrictByDefaultMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat());
    IMetaStoreClient.NotificationFilter restrictByArbitraryMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat() + "_bogus");
    NotificationEvent dummyEvent = createDummyEvent(dbname, tblname, 0);
    assertEquals(MessageFactory.getInstance().getMessageFormat(), dummyEvent.getMessageFormat());
    assertFalse(restrictByDefaultMessageFormat.accept(null));
    assertTrue(restrictByDefaultMessageFormat.accept(dummyEvent));
    assertFalse(restrictByArbitraryMessageFormat.accept(dummyEvent));
    // Test andFilter operation.
    IMetaStoreClient.NotificationFilter yes = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return true;
        }
    };
    IMetaStoreClient.NotificationFilter no = new IMetaStoreClient.NotificationFilter() {

        @Override
        public boolean accept(NotificationEvent notificationEvent) {
            return false;
        }
    };
    assertTrue(new AndFilter(yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no).accept(dummyEvent));
    assertTrue(new AndFilter(yes, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(yes, no, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, yes, no).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, yes).accept(dummyEvent));
    assertFalse(new AndFilter(no, no, no).accept(dummyEvent));
}

Example 2 with DatabaseAndTableFilter

use of org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter in project hive by apache.

the class ReplDumpTask method incrementalDump.

private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // get list of events matching dbPattern & tblPattern
    Long lastReplId;
    // go through each event, and dump out each event to a event-level dump dir inside dumproot
    // TODO : instead of simply restricting by message format, we should eventually
    // move to a jdbc-driver-stype registering of message format, and picking message
    // factory per event to decode. For now, however, since all messages have the
    // same factory, restricting by message format is effectively a guard against
    // older leftover data that would cause us problems.
    work.overrideEventTo(getHive());
    IMetaStoreClient.NotificationFilter evFilter = new AndFilter(new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern), new EventBoundaryFilter(work.eventFrom, work.eventTo), new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat()));
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(getHive().getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, work.eventFrom, work.maxEventLimit(), evFilter);
    lastReplId = work.eventTo;
    String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty()) ? work.dbNameOrPattern : "?";
    replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(), evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
    replLogger.startLog();
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        lastReplId = ev.getEventId();
        Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
        dumpEvent(ev, evRoot, cmRoot);
    }
    replLogger.endLog(lastReplId.toString());
    LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
    Utils.writeOutput(Arrays.asList("incremental", String.valueOf(work.eventFrom), String.valueOf(lastReplId)), dmd.getDumpFilePath(), conf);
    dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot);
    dmd.write();
    return lastReplId;
}

Example 3 with DatabaseAndTableFilter

use of org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter in project hive by apache.

the class ReplDumpTask method bootStrapDump.

private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // bootstrap case
    Hive hiveDb = getHive();
    Long bootDumpBeginReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    for (String dbName : Utils.matchesDb(hiveDb, work.dbNameOrPattern)) {
        LOG.debug("ReplicationSemanticAnalyzer: analyzeReplDump dumping db: " + dbName);
        replLogger = new BootstrapDumpLogger(dbName, dumpRoot.toString(), Utils.getAllTables(getHive(), dbName).size(), getHive().getAllFunctions().size());
        replLogger.startLog();
        Path dbRoot = dumpDbMetadata(dbName, dumpRoot);
        dumpFunctionMetadata(dbName, dumpRoot);
        String uniqueKey = Utils.setDbBootstrapDumpState(hiveDb, dbName);
        for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.tableNameOrPattern)) {
            LOG.debug("analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri());
            dumpTable(dbName, tblName, dbRoot);
            dumpConstraintMetadata(dbName, tblName, dbRoot);
        }
        Utils.resetDbBootstrapDumpState(hiveDb, dbName, uniqueKey);
        replLogger.endLog(bootDumpBeginReplId.toString());
    }
    Long bootDumpEndReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    LOG.info("Bootstrap object dump phase took from {} to {}", bootDumpBeginReplId, bootDumpEndReplId);
    // Now that bootstrap has dumped all objects related, we have to account for the changes
    // that occurred while bootstrap was happening - i.e. we have to look through all events
    // during the bootstrap period and consolidate them with our dump.
    IMetaStoreClient.NotificationFilter evFilter = new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern);
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(hiveDb.getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, bootDumpBeginReplId, Ints.checkedCast(bootDumpEndReplId - bootDumpBeginReplId) + 1, evFilter);
    // Now we consolidate all the events that happenned during the objdump into the objdump
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        Path eventRoot = new Path(dumpRoot, String.valueOf(ev.getEventId()));
    // FIXME : implement consolidateEvent(..) similar to dumpEvent(ev,evRoot)
    }
    LOG.info("Consolidation done, preparing to return {},{}->{}", dumpRoot.toUri(), bootDumpBeginReplId, bootDumpEndReplId);
    dmd.setDump(DumpType.BOOTSTRAP, bootDumpBeginReplId, bootDumpEndReplId, cmRoot);
    dmd.write();
    // Currently returned bootDumpBeginReplId as we don't consolidate the events after bootstrap
    return bootDumpBeginReplId;
}