use of org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter in project hive by apache.
the class TestReplicationScenarios method testEventFilters.
@Test
public void testEventFilters() {
// Test testing that the filters introduced by EventUtils are working correctly.
// The current filters we use in ReplicationSemanticAnalyzer is as follows:
// IMetaStoreClient.NotificationFilter evFilter = EventUtils.andFilter(
// EventUtils.getDbTblNotificationFilter(dbNameOrPattern, tblNameOrPattern),
// EventUtils.getEventBoundaryFilter(eventFrom, eventTo),
// EventUtils.restrictByMessageFormat(MessageFactory.getInstance().getMessageFormat()));
// So, we test each of those three filters, and then test andFilter itself.
String dbname = "testfilter_db";
String tblname = "testfilter_tbl";
// Test EventUtils.getDbTblNotificationFilter - this is supposed to restrict
// events to those that match the dbname and tblname provided to the filter.
// If the tblname passed in to the filter is null, then it restricts itself
// to dbname-matching alone.
IMetaStoreClient.NotificationFilter dbTblFilter = new DatabaseAndTableFilter(dbname, tblname);
IMetaStoreClient.NotificationFilter dbFilter = new DatabaseAndTableFilter(dbname, null);
assertFalse(dbTblFilter.accept(null));
assertTrue(dbTblFilter.accept(createDummyEvent(dbname, tblname, 0)));
assertFalse(dbTblFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
assertFalse(dbTblFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
assertFalse(dbFilter.accept(null));
assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname, 0)));
assertTrue(dbFilter.accept(createDummyEvent(dbname, tblname + "extra", 0)));
assertFalse(dbFilter.accept(createDummyEvent(dbname + "extra", tblname, 0)));
// Test EventUtils.getEventBoundaryFilter - this is supposed to only allow events
// within a range specified.
long evBegin = 50;
long evEnd = 75;
IMetaStoreClient.NotificationFilter evRangeFilter = new EventBoundaryFilter(evBegin, evEnd);
assertTrue(evBegin < evEnd);
assertFalse(evRangeFilter.accept(null));
assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin - 1)));
assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin)));
assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evBegin + 1)));
assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd - 1)));
assertTrue(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd)));
assertFalse(evRangeFilter.accept(createDummyEvent(dbname, tblname, evEnd + 1)));
// Test EventUtils.restrictByMessageFormat - this restricts events generated to those
// that match a provided message format
IMetaStoreClient.NotificationFilter restrictByDefaultMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat());
IMetaStoreClient.NotificationFilter restrictByArbitraryMessageFormat = new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat() + "_bogus");
NotificationEvent dummyEvent = createDummyEvent(dbname, tblname, 0);
assertEquals(MessageFactory.getInstance().getMessageFormat(), dummyEvent.getMessageFormat());
assertFalse(restrictByDefaultMessageFormat.accept(null));
assertTrue(restrictByDefaultMessageFormat.accept(dummyEvent));
assertFalse(restrictByArbitraryMessageFormat.accept(dummyEvent));
// Test andFilter operation.
IMetaStoreClient.NotificationFilter yes = new IMetaStoreClient.NotificationFilter() {
@Override
public boolean accept(NotificationEvent notificationEvent) {
return true;
}
};
IMetaStoreClient.NotificationFilter no = new IMetaStoreClient.NotificationFilter() {
@Override
public boolean accept(NotificationEvent notificationEvent) {
return false;
}
};
assertTrue(new AndFilter(yes, yes).accept(dummyEvent));
assertFalse(new AndFilter(yes, no).accept(dummyEvent));
assertFalse(new AndFilter(no, yes).accept(dummyEvent));
assertFalse(new AndFilter(no, no).accept(dummyEvent));
assertTrue(new AndFilter(yes, yes, yes).accept(dummyEvent));
assertFalse(new AndFilter(yes, yes, no).accept(dummyEvent));
assertFalse(new AndFilter(yes, no, yes).accept(dummyEvent));
assertFalse(new AndFilter(yes, no, no).accept(dummyEvent));
assertFalse(new AndFilter(no, yes, yes).accept(dummyEvent));
assertFalse(new AndFilter(no, yes, no).accept(dummyEvent));
assertFalse(new AndFilter(no, no, yes).accept(dummyEvent));
assertFalse(new AndFilter(no, no, no).accept(dummyEvent));
}
use of org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter in project hive by apache.
the class ReplDumpTask method incrementalDump.
private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
// get list of events matching dbPattern & tblPattern
Long lastReplId;
// go through each event, and dump out each event to a event-level dump dir inside dumproot
// TODO : instead of simply restricting by message format, we should eventually
// move to a jdbc-driver-stype registering of message format, and picking message
// factory per event to decode. For now, however, since all messages have the
// same factory, restricting by message format is effectively a guard against
// older leftover data that would cause us problems.
work.overrideEventTo(getHive());
IMetaStoreClient.NotificationFilter evFilter = new AndFilter(new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern), new EventBoundaryFilter(work.eventFrom, work.eventTo), new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat()));
EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(getHive().getMSC());
EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, work.eventFrom, work.maxEventLimit(), evFilter);
lastReplId = work.eventTo;
String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty()) ? work.dbNameOrPattern : "?";
replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(), evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
replLogger.startLog();
while (evIter.hasNext()) {
NotificationEvent ev = evIter.next();
lastReplId = ev.getEventId();
Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
dumpEvent(ev, evRoot, cmRoot);
}
replLogger.endLog(lastReplId.toString());
LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
Utils.writeOutput(Arrays.asList("incremental", String.valueOf(work.eventFrom), String.valueOf(lastReplId)), dmd.getDumpFilePath(), conf);
dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot);
dmd.write();
return lastReplId;
}
Aggregations