Examples with NotificationEvent org.apache.hadoop.hive.metastore.api.NotificationEvent used on opensource projects

Example 61 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAlterPartition.

/**
 * @param partitionEvent partition event
 * @throws MetaException
 */
@Override
public void onAlterPartition(AlterPartitionEvent partitionEvent) throws MetaException {
    Partition before = partitionEvent.getOldPartition();
    Partition after = partitionEvent.getNewPartition();
    NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_PARTITION.toString(), msgFactory.buildAlterPartitionMessage(partitionEvent.getTable(), before, after, partitionEvent.getIsTruncateOp()).toString());
    event.setDbName(before.getDbName());
    event.setTableName(before.getTableName());
    process(event, partitionEvent);
}

Example 62 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAddForeignKey.

/**
 * @param addForeignKeyEvent add foreign key event
 * @throws MetaException
 */
@Override
public void onAddForeignKey(AddForeignKeyEvent addForeignKeyEvent) throws MetaException {
    List<SQLForeignKey> cols = addForeignKeyEvent.getForeignKeyCols();
    if (cols.size() > 0) {
        NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_FOREIGNKEY.toString(), msgFactory.buildAddForeignKeyMessage(addForeignKeyEvent.getForeignKeyCols()).toString());
        event.setDbName(cols.get(0).getPktable_db());
        event.setTableName(cols.get(0).getPktable_name());
        process(event, addForeignKeyEvent);
    }
}

Example 63 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class DbNotificationListener method onAddNotNullConstraint.

/**
 * @param addNotNullConstraintEvent add not null constraint event
 * @throws MetaException
 */
@Override
public void onAddNotNullConstraint(AddNotNullConstraintEvent addNotNullConstraintEvent) throws MetaException {
    List<SQLNotNullConstraint> cols = addNotNullConstraintEvent.getNotNullConstraintCols();
    if (cols.size() > 0) {
        NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_NOTNULLCONSTRAINT.toString(), msgFactory.buildAddNotNullConstraintMessage(addNotNullConstraintEvent.getNotNullConstraintCols()).toString());
        event.setDbName(cols.get(0).getTable_db());
        event.setTableName(cols.get(0).getTable_name());
        process(event, addNotNullConstraintEvent);
    }
}

Example 64 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class ReplDumpTask method incrementalDump.

private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // get list of events matching dbPattern & tblPattern
    Long lastReplId;
    // go through each event, and dump out each event to a event-level dump dir inside dumproot
    // TODO : instead of simply restricting by message format, we should eventually
    // move to a jdbc-driver-stype registering of message format, and picking message
    // factory per event to decode. For now, however, since all messages have the
    // same factory, restricting by message format is effectively a guard against
    // older leftover data that would cause us problems.
    work.overrideEventTo(getHive());
    IMetaStoreClient.NotificationFilter evFilter = new AndFilter(new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern), new EventBoundaryFilter(work.eventFrom, work.eventTo), new MessageFormatFilter(MessageFactory.getInstance().getMessageFormat()));
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(getHive().getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, work.eventFrom, work.maxEventLimit(), evFilter);
    lastReplId = work.eventTo;
    String dbName = (null != work.dbNameOrPattern && !work.dbNameOrPattern.isEmpty()) ? work.dbNameOrPattern : "?";
    replLogger = new IncrementalDumpLogger(dbName, dumpRoot.toString(), evFetcher.getDbNotificationEventsCount(work.eventFrom, dbName));
    replLogger.startLog();
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        lastReplId = ev.getEventId();
        Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId));
        dumpEvent(ev, evRoot, cmRoot);
    }
    replLogger.endLog(lastReplId.toString());
    LOG.info("Done dumping events, preparing to return {},{}", dumpRoot.toUri(), lastReplId);
    Utils.writeOutput(Arrays.asList("incremental", String.valueOf(work.eventFrom), String.valueOf(lastReplId)), dmd.getDumpFilePath(), conf);
    dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot);
    dmd.write();
    return lastReplId;
}

Example 65 with NotificationEvent

use of org.apache.hadoop.hive.metastore.api.NotificationEvent in project hive by apache.

the class ReplDumpTask method bootStrapDump.

private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
    // bootstrap case
    Hive hiveDb = getHive();
    Long bootDumpBeginReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    for (String dbName : Utils.matchesDb(hiveDb, work.dbNameOrPattern)) {
        LOG.debug("ReplicationSemanticAnalyzer: analyzeReplDump dumping db: " + dbName);
        replLogger = new BootstrapDumpLogger(dbName, dumpRoot.toString(), Utils.getAllTables(getHive(), dbName).size(), getHive().getAllFunctions().size());
        replLogger.startLog();
        Path dbRoot = dumpDbMetadata(dbName, dumpRoot);
        dumpFunctionMetadata(dbName, dumpRoot);
        String uniqueKey = Utils.setDbBootstrapDumpState(hiveDb, dbName);
        for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.tableNameOrPattern)) {
            LOG.debug("analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri());
            dumpTable(dbName, tblName, dbRoot);
            dumpConstraintMetadata(dbName, tblName, dbRoot);
        }
        Utils.resetDbBootstrapDumpState(hiveDb, dbName, uniqueKey);
        replLogger.endLog(bootDumpBeginReplId.toString());
    }
    Long bootDumpEndReplId = hiveDb.getMSC().getCurrentNotificationEventId().getEventId();
    LOG.info("Bootstrap object dump phase took from {} to {}", bootDumpBeginReplId, bootDumpEndReplId);
    // Now that bootstrap has dumped all objects related, we have to account for the changes
    // that occurred while bootstrap was happening - i.e. we have to look through all events
    // during the bootstrap period and consolidate them with our dump.
    IMetaStoreClient.NotificationFilter evFilter = new DatabaseAndTableFilter(work.dbNameOrPattern, work.tableNameOrPattern);
    EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(hiveDb.getMSC());
    EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, bootDumpBeginReplId, Ints.checkedCast(bootDumpEndReplId - bootDumpBeginReplId) + 1, evFilter);
    // Now we consolidate all the events that happenned during the objdump into the objdump
    while (evIter.hasNext()) {
        NotificationEvent ev = evIter.next();
        Path eventRoot = new Path(dumpRoot, String.valueOf(ev.getEventId()));
    // FIXME : implement consolidateEvent(..) similar to dumpEvent(ev,evRoot)
    }
    LOG.info("Consolidation done, preparing to return {},{}->{}", dumpRoot.toUri(), bootDumpBeginReplId, bootDumpEndReplId);
    dmd.setDump(DumpType.BOOTSTRAP, bootDumpBeginReplId, bootDumpEndReplId, cmRoot);
    dmd.write();
    // Currently returned bootDumpBeginReplId as we don't consolidate the events after bootstrap
    return bootDumpBeginReplId;
}