Examples with EventBatchList org.apache.hadoop.hdfs.inotify.EventBatchList used on opensource projects

Example 1 with EventBatchList

use of org.apache.hadoop.hdfs.inotify.EventBatchList in project hadoop by apache.

the class NameNodeRpcServer method getEditsFromTxid.

// ClientProtocol
@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
    checkNNStartup();
    // only active
    namesystem.checkOperation(OperationCategory.READ);
    namesystem.checkSuperuserPrivilege();
    int maxEventsPerRPC = nn.getConf().getInt(DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY, DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
    FSEditLog log = namesystem.getFSImage().getEditLog();
    long syncTxid = log.getSyncTxId();
    // If we haven't synced anything yet, we can only read finalized
    // segments since we can't reliably determine which txns in in-progress
    // segments have actually been committed (e.g. written to a quorum of JNs).
    // If we have synced txns, we can definitely read up to syncTxid since
    // syncTxid is only updated after a transaction is committed to all
    // journals. (In-progress segments written by old writers are already
    // discarded for us, so if we read any in-progress segments they are
    // guaranteed to have been written by this NameNode.)
    boolean readInProgress = syncTxid > 0;
    List<EventBatch> batches = Lists.newArrayList();
    int totalEvents = 0;
    long maxSeenTxid = -1;
    long firstSeenTxid = -1;
    if (syncTxid > 0 && txid > syncTxid) {
        // we can't read past syncTxid, so there's no point in going any further
        return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
    }
    Collection<EditLogInputStream> streams = null;
    try {
        streams = log.selectInputStreams(txid, 0, null, readInProgress);
    } catch (IllegalStateException e) {
        // can happen if we have
        // transitioned out of active and haven't yet transitioned to standby
        // and are using QJM -- the edit log will be closed and this exception
        // will result
        LOG.info("NN is transitioning from active to standby and FSEditLog " + "is closed -- could not read edits");
        return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
    }
    boolean breakOuter = false;
    for (EditLogInputStream elis : streams) {
        // starting txid
        try {
            FSEditLogOp op = null;
            while ((op = readOp(elis)) != null) {
                // txid we get is greater than syncTxid
                if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
                    breakOuter = true;
                    break;
                }
                EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
                if (eventBatch != null) {
                    batches.add(eventBatch);
                    totalEvents += eventBatch.getEvents().length;
                }
                if (op.getTransactionId() > maxSeenTxid) {
                    maxSeenTxid = op.getTransactionId();
                }
                if (firstSeenTxid == -1) {
                    firstSeenTxid = op.getTransactionId();
                }
                if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 && op.getTransactionId() == syncTxid)) {
                    // we're done
                    breakOuter = true;
                    break;
                }
            }
        } finally {
            elis.close();
        }
        if (breakOuter) {
            break;
        }
    }
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}

Example 2 with EventBatchList

use of org.apache.hadoop.hdfs.inotify.EventBatchList in project hadoop by apache.

the class DFSInotifyEventInputStream method poll.

/**
   * Returns the next batch of events in the stream or null if no new
   * batches are currently available.
   *
   * @throws IOException because of network error or edit log
   * corruption. Also possible if JournalNodes are unresponsive in the
   * QJM setting (even one unresponsive JournalNode is enough in rare cases),
   * so catching this exception and retrying at least a few times is
   * recommended.
   * @throws MissingEventsException if we cannot return the next batch in the
   * stream because the data for the events (and possibly some subsequent
   * events) has been deleted (generally because this stream is a very large
   * number of transactions behind the current state of the NameNode). It is
   * safe to continue reading from the stream after this exception is thrown
   * The next available batch of events will be returned.
   */
public EventBatch poll() throws IOException, MissingEventsException {
    try (TraceScope ignored = tracer.newScope("inotifyPoll")) {
        // need to keep retrying until the NN sends us the latest committed txid
        if (lastReadTxid == -1) {
            LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
            lastReadTxid = namenode.getCurrentEditLogTxid();
            return null;
        }
        if (!it.hasNext()) {
            EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
            if (el.getLastTxid() != -1) {
                // we only want to set syncTxid when we were actually able to read some
                // edits on the NN -- otherwise it will seem like edits are being
                // generated faster than we can read them when the problem is really
                // that we are temporarily unable to read edits
                syncTxid = el.getSyncTxid();
                it = el.getBatches().iterator();
                long formerLastReadTxid = lastReadTxid;
                lastReadTxid = el.getLastTxid();
                if (el.getFirstTxid() != formerLastReadTxid + 1) {
                    throw new MissingEventsException(formerLastReadTxid + 1, el.getFirstTxid());
                }
            } else {
                LOG.debug("poll(): read no edits from the NN when requesting edits " + "after txid {}", lastReadTxid);
                return null;
            }
        }
        if (it.hasNext()) {
            // newly seen edit log ops actually got converted to events
            return it.next();
        } else {
            return null;
        }
    }
}

Example 3 with EventBatchList

use of org.apache.hadoop.hdfs.inotify.EventBatchList in project hadoop by apache.

the class PBHelperClient method convert.

public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws IOException {
    final InotifyProtos.EventsListProto list = resp.getEventsList();
    final long firstTxid = list.getFirstTxid();
    final long lastTxid = list.getLastTxid();
    List<EventBatch> batches = Lists.newArrayList();
    if (list.getEventsList().size() > 0) {
        throw new IOException("Can't handle old inotify server response.");
    }
    for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
        long txid = bp.getTxid();
        if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
            throw new IOException("Error converting TxidResponseProto: got a " + "transaction id " + txid + " that was outside the range of [" + firstTxid + ", " + lastTxid + "].");
        }
        List<Event> events = Lists.newArrayList();
        for (InotifyProtos.EventProto p : bp.getEventsList()) {
            switch(p.getType()) {
                case EVENT_CLOSE:
                    InotifyProtos.CloseEventProto close = InotifyProtos.CloseEventProto.parseFrom(p.getContents());
                    events.add(new Event.CloseEvent(close.getPath(), close.getFileSize(), close.getTimestamp()));
                    break;
                case EVENT_CREATE:
                    InotifyProtos.CreateEventProto create = InotifyProtos.CreateEventProto.parseFrom(p.getContents());
                    events.add(new Event.CreateEvent.Builder().iNodeType(createTypeConvert(create.getType())).path(create.getPath()).ctime(create.getCtime()).ownerName(create.getOwnerName()).groupName(create.getGroupName()).perms(convert(create.getPerms())).replication(create.getReplication()).symlinkTarget(create.getSymlinkTarget().isEmpty() ? null : create.getSymlinkTarget()).defaultBlockSize(create.getDefaultBlockSize()).overwrite(create.getOverwrite()).build());
                    break;
                case EVENT_METADATA:
                    InotifyProtos.MetadataUpdateEventProto meta = InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
                    events.add(new Event.MetadataUpdateEvent.Builder().path(meta.getPath()).metadataType(metadataUpdateTypeConvert(meta.getType())).mtime(meta.getMtime()).atime(meta.getAtime()).replication(meta.getReplication()).ownerName(meta.getOwnerName().isEmpty() ? null : meta.getOwnerName()).groupName(meta.getGroupName().isEmpty() ? null : meta.getGroupName()).perms(meta.hasPerms() ? convert(meta.getPerms()) : null).acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(meta.getAclsList())).xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(meta.getXAttrsList())).xAttrsRemoved(meta.getXAttrsRemoved()).build());
                    break;
                case EVENT_RENAME:
                    InotifyProtos.RenameEventProto rename = InotifyProtos.RenameEventProto.parseFrom(p.getContents());
                    events.add(new Event.RenameEvent.Builder().srcPath(rename.getSrcPath()).dstPath(rename.getDestPath()).timestamp(rename.getTimestamp()).build());
                    break;
                case EVENT_APPEND:
                    InotifyProtos.AppendEventProto append = InotifyProtos.AppendEventProto.parseFrom(p.getContents());
                    events.add(new Event.AppendEvent.Builder().path(append.getPath()).newBlock(append.hasNewBlock() && append.getNewBlock()).build());
                    break;
                case EVENT_UNLINK:
                    InotifyProtos.UnlinkEventProto unlink = InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
                    events.add(new Event.UnlinkEvent.Builder().path(unlink.getPath()).timestamp(unlink.getTimestamp()).build());
                    break;
                case EVENT_TRUNCATE:
                    InotifyProtos.TruncateEventProto truncate = InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
                    events.add(new Event.TruncateEvent(truncate.getPath(), truncate.getFileSize(), truncate.getTimestamp()));
                    break;
                default:
                    throw new RuntimeException("Unexpected inotify event type: " + p.getType());
            }
        }
        batches.add(new EventBatch(txid, events.toArray(new Event[0])));
    }
    return new EventBatchList(batches, resp.getEventsList().getFirstTxid(), resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}