Examples with BootstrapDumpLogger org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger used on opensource projects

Example 1 with BootstrapDumpLogger

use of org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger in project hive by apache.

the class ReplDumpTask method bootStrapDump.

private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // bootstrap case
    Hive hiveDb = getHive();
    Long bootDumpBeginReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    for (String dbName : Utils.matchesDb(hiveDb, work.dbNameOrPattern)) {
        LOG.debug("ReplicationSemanticAnalyzer: analyzeReplDump dumping db: " + dbName);
        replLogger = new BootstrapDumpLogger(dbName, dumpRoot.toString(), Utils.getAllTables(getHive(), dbName).size(), getHive().getAllFunctions().size());
        replLogger.startLog();
        Path dbRoot = dumpDbMetadata(dbName, dumpRoot);
        dumpFunctionMetadata(dbName, dumpRoot);
        String uniqueKey = Utils.setDbBootstrapDumpState(hiveDb, dbName);
        for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.tableNameOrPattern)) {
            LOG.debug("analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri());
            dumpTable(dbName, tblName, dbRoot);
            dumpConstraintMetadata(dbName, tblName, dbRoot);
        }
        Utils.resetDbBootstrapDumpState(hiveDb, dbName, uniqueKey);
        replLogger.endLog(bootDumpBeginReplId.toString());
    }
    Long bootDumpEndReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    LOG.info("Bootstrap object dump phase took from {} to {}", bootDumpBeginReplId, bootDumpEndReplId);
    // Now that bootstrap has dumped all objects related, we have to account for the changes
    // that occurred while bootstrap was happening - i.e. we have to look through all events
    // during the bootstrap period and consolidate them with our dump.
    IMetaStoreClient.NotificationFilter evFilter = new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern);
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(hiveDb.getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, bootDumpBeginReplId, Ints.checkedCast(bootDumpEndReplId - bootDumpBeginReplId) + 1, evFilter);
    // Now we consolidate all the events that happenned during the objdump into the objdump
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        Path eventRoot = new Path(dumpRoot, String.valueOf(ev.getEventId()));
    // FIXME : implement consolidateEvent(..) similar to dumpEvent(ev,evRoot)
    }
    LOG.info("Consolidation done, preparing to return {},{}->{}", dumpRoot.toUri(), bootDumpBeginReplId, bootDumpEndReplId);
    dmd.setDump(DumpType.BOOTSTRAP, bootDumpBeginReplId, bootDumpEndReplId, cmRoot);
    dmd.write();
    // Currently returned bootDumpBeginReplId as we don't consolidate the events after bootstrap
    return bootDumpBeginReplId;
}