Examples with IncrementalDumpLogger org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger used on opensource projects

Example 1 with IncrementalDumpLogger

use of org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger in project hive by apache.

the class ReplDumpTask method incrementalDump.

private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // get list of events matching dbPattern & tblPattern
    Long lastReplId;
    // go through each event, and dump out each event to a event-level dump dir inside dumproot
    // TODO : instead of simply restricting by message format, we should eventually
    // move to a jdbc-driver-stype registering of message format, and picking message
    // factory per event to decode. For now, however, since all messages have the
    // same factory, restricting by message format is effectively a guard against
    // older leftover data that would cause us problems.
    work.overrideEventTo(getHive());
    IMetaStoreClient.NotificationFilter evFilter = new AndFilter(new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern), new EventBoundaryFilter(work.eventFrom, work.eventTo), new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat()));
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(getHive().getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, work.eventFrom, work.maxEventLimit(), evFilter);
    lastReplId = work.eventTo;
    String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty()) ? work.dbNameOrPattern : "?";
    replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(), evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
    replLogger.startLog();
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        lastReplId = ev.getEventId();
        Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
        dumpEvent(ev, evRoot, cmRoot);
    }
    replLogger.endLog(lastReplId.toString());
    LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
    Utils.writeOutput(Arrays.asList("incremental", String.valueOf(work.eventFrom), String.valueOf(lastReplId)), dmd.getDumpFilePath(), conf);
    dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot);
    dmd.write();
    return lastReplId;
}