Examples with Event org.apache.hadoop.hdfs.inotify.Event used on opensource projects

Example 11 with Event

use of org.apache.hadoop.hdfs.inotify.Event in project alluxio by Alluxio.

the class SupportedHdfsActiveSyncProvider method pollEvent.

/**
 * Fetch and process events.
 * @param eventStream event stream
 */
public void pollEvent(DFSInotifyEventInputStream eventStream) {
    LOG.debug("Polling thread starting, with timeout {} ms", mActiveUfsPollTimeoutMs);
    long start = System.currentTimeMillis();
    long behind = eventStream.getTxidsBehindEstimate();
    while (!Thread.currentThread().isInterrupted()) {
        try {
            List<Callable<Integer>> process = new LinkedList<>();
            for (int i = 0; i < mBatchSize; i++) {
                EventBatch batch = eventStream.poll(mActiveUfsPollTimeoutMs, TimeUnit.MILLISECONDS);
                if (batch == null) {
                    break;
                }
                process.add(() -> {
                    for (Event event : batch.getEvents()) {
                        processEvent(event, mUfsUriList, batch.getTxid());
                    }
                    return batch.getEvents().length;
                });
            }
            mProcessTasks.add(mExecutorService.submit(() -> process.stream().map(callable -> {
                try {
                    return callable.call();
                } catch (Exception e) {
                    LogUtils.warnWithException(LOG, "Failed to process event", e);
                    return 0;
                }
            }).reduce(0, Integer::sum)));
            long end = System.currentTimeMillis();
            if (end > (start + mActiveUfsSyncEventRateInterval)) {
                long currentlyBehind = eventStream.getTxidsBehindEstimate();
                long processedEvents = getCountSinceLastLog();
                long hdfsEvents = processedEvents + currentlyBehind - behind;
                long durationMs = end - start;
                if (LOG.isDebugEnabled()) {
                    // for debug, print every interval
                    LOG.debug("HDFS sync stats. past duration: {} ms. HDFS generated events: {} ({} events/s). " + "Processed events: {} ({} events/s). TxidsBehindEstimate: {}", durationMs, hdfsEvents, String.format("%.2f", hdfsEvents * 1000.0f / durationMs), processedEvents, String.format("%.2f", processedEvents * 1000.0f / durationMs), currentlyBehind);
                } else {
                    // for info, print with the sampling logger
                    SAMPLING_LOG.info("HDFS sync stats. past duration: {} ms. HDFS generated events: {} ({} events/s). " + "Processed events: {} ({} events/s). TxidsBehindEstimate: {}", durationMs, hdfsEvents, String.format("%.2f", hdfsEvents * 1000.0f / durationMs), processedEvents, String.format("%.2f", processedEvents * 1000.0f / durationMs), currentlyBehind);
                }
                behind = currentlyBehind;
                start = end;
            }
        } catch (IOException e) {
            LOG.warn("IOException occured during polling inotify: {}", e.toString());
            if (e.getCause() instanceof InterruptedException) {
                return;
            }
        } catch (MissingEventsException e) {
            LOG.warn("MissingEventException during polling: {}", e.toString());
            mEventMissed = true;
        // need to sync all syncpoints at this point
        } catch (InterruptedException e) {
            LOG.warn("InterruptedException during polling: {}", e.toString());
            return;
        }
    }
}

Example 12 with Event

use of org.apache.hadoop.hdfs.inotify.Event in project alluxio by Alluxio.

the class SupportedHdfsActiveSyncProvider method processEvent.

private boolean processEvent(Event event, List<AlluxioURI> syncPointList, long txId) {
    boolean fileMatch = false;
    String filePath = "";
    String renameFilePath = "";
    switch(event.getEventType()) {
        case CREATE:
            Event.CreateEvent createEvent = (Event.CreateEvent) event;
            filePath = createEvent.getPath();
            break;
        case UNLINK:
            Event.UnlinkEvent unlinkEvent = (Event.UnlinkEvent) event;
            filePath = unlinkEvent.getPath();
            break;
        case APPEND:
            Event.AppendEvent appendEvent = (Event.AppendEvent) event;
            filePath = appendEvent.getPath();
            break;
        case RENAME:
            Event.RenameEvent renameEvent = (Event.RenameEvent) event;
            filePath = renameEvent.getSrcPath();
            renameFilePath = renameEvent.getDstPath();
            break;
        case METADATA:
            Event.MetadataUpdateEvent metadataUpdateEvent = (Event.MetadataUpdateEvent) event;
            filePath = metadataUpdateEvent.getPath();
            break;
        default:
            break;
    }
    if (filePath.isEmpty()) {
        return false;
    }
    for (AlluxioURI syncPoint : syncPointList) {
        try {
            // find out if the changed file falls under one of the sync points
            if (PathUtils.hasPrefix(filePath, syncPoint.getPath())) {
                fileMatch = true;
                recordFileChanged(syncPoint.toString(), filePath, txId);
            }
        } catch (InvalidPathException e) {
            LOG.info("Invalid path encountered {} ", filePath);
        }
        try {
            // find out if the changed file falls under one of the sync points
            if ((!renameFilePath.isEmpty()) && PathUtils.hasPrefix(renameFilePath, syncPoint.getPath())) {
                fileMatch = true;
                recordFileChanged(syncPoint.toString(), renameFilePath, txId);
            }
        } catch (InvalidPathException e) {
            LOG.info("Invalid path encountered {} ", renameFilePath);
        }
    }
    try (LockResource r = new LockResource(mWriteLock)) {
        mCurrentTxId = txId;
    }
    return fileMatch;
}

Example 13 with Event

use of org.apache.hadoop.hdfs.inotify.Event in project hadoop by apache.

the class PBHelperClient method convertEditsResponse.

public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
    InotifyProtos.EventsListProto.Builder builder = InotifyProtos.EventsListProto.newBuilder();
    for (EventBatch b : el.getBatches()) {
        List<InotifyProtos.EventProto> events = Lists.newArrayList();
        for (Event e : b.getEvents()) {
            switch(e.getEventType()) {
                case CLOSE:
                    Event.CloseEvent ce = (Event.CloseEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_CLOSE).setContents(InotifyProtos.CloseEventProto.newBuilder().setPath(ce.getPath()).setFileSize(ce.getFileSize()).setTimestamp(ce.getTimestamp()).build().toByteString()).build());
                    break;
                case CREATE:
                    Event.CreateEvent ce2 = (Event.CreateEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_CREATE).setContents(InotifyProtos.CreateEventProto.newBuilder().setType(createTypeConvert(ce2.getiNodeType())).setPath(ce2.getPath()).setCtime(ce2.getCtime()).setOwnerName(ce2.getOwnerName()).setGroupName(ce2.getGroupName()).setPerms(convert(ce2.getPerms())).setReplication(ce2.getReplication()).setSymlinkTarget(ce2.getSymlinkTarget() == null ? "" : ce2.getSymlinkTarget()).setDefaultBlockSize(ce2.getDefaultBlockSize()).setOverwrite(ce2.getOverwrite()).build().toByteString()).build());
                    break;
                case METADATA:
                    Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
                    InotifyProtos.MetadataUpdateEventProto.Builder metaB = InotifyProtos.MetadataUpdateEventProto.newBuilder().setPath(me.getPath()).setType(metadataUpdateTypeConvert(me.getMetadataType())).setMtime(me.getMtime()).setAtime(me.getAtime()).setReplication(me.getReplication()).setOwnerName(me.getOwnerName() == null ? "" : me.getOwnerName()).setGroupName(me.getGroupName() == null ? "" : me.getGroupName()).addAllAcls(me.getAcls() == null ? Lists.<AclEntryProto>newArrayList() : convertAclEntryProto(me.getAcls())).addAllXAttrs(me.getxAttrs() == null ? Lists.<XAttrProto>newArrayList() : convertXAttrProto(me.getxAttrs())).setXAttrsRemoved(me.isxAttrsRemoved());
                    if (me.getPerms() != null) {
                        metaB.setPerms(convert(me.getPerms()));
                    }
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_METADATA).setContents(metaB.build().toByteString()).build());
                    break;
                case RENAME:
                    Event.RenameEvent re = (Event.RenameEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_RENAME).setContents(InotifyProtos.RenameEventProto.newBuilder().setSrcPath(re.getSrcPath()).setDestPath(re.getDstPath()).setTimestamp(re.getTimestamp()).build().toByteString()).build());
                    break;
                case APPEND:
                    Event.AppendEvent re2 = (Event.AppendEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_APPEND).setContents(InotifyProtos.AppendEventProto.newBuilder().setPath(re2.getPath()).setNewBlock(re2.toNewBlock()).build().toByteString()).build());
                    break;
                case UNLINK:
                    Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_UNLINK).setContents(InotifyProtos.UnlinkEventProto.newBuilder().setPath(ue.getPath()).setTimestamp(ue.getTimestamp()).build().toByteString()).build());
                    break;
                case TRUNCATE:
                    Event.TruncateEvent te = (Event.TruncateEvent) e;
                    events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_TRUNCATE).setContents(InotifyProtos.TruncateEventProto.newBuilder().setPath(te.getPath()).setFileSize(te.getFileSize()).setTimestamp(te.getTimestamp()).build().toByteString()).build());
                    break;
                default:
                    throw new RuntimeException("Unexpected inotify event: " + e);
            }
        }
        builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().setTxid(b.getTxid()).addAllEvents(events));
    }
    builder.setFirstTxid(el.getFirstTxid());
    builder.setLastTxid(el.getLastTxid());
    builder.setSyncTxid(el.getSyncTxid());
    return GetEditsFromTxidResponseProto.newBuilder().setEventsList(builder.build()).build();
}

Example 14 with Event

use of org.apache.hadoop.hdfs.inotify.Event in project hadoop by apache.

the class PBHelperClient method convert.

public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws IOException {
    final InotifyProtos.EventsListProto list = resp.getEventsList();
    final long firstTxid = list.getFirstTxid();
    final long lastTxid = list.getLastTxid();
    List<EventBatch> batches = Lists.newArrayList();
    if (list.getEventsList().size() > 0) {
        throw new IOException("Can't handle old inotify server response.");
    }
    for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
        long txid = bp.getTxid();
        if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
            throw new IOException("Error converting TxidResponseProto: got a " + "transaction id " + txid + " that was outside the range of [" + firstTxid + ", " + lastTxid + "].");
        }
        List<Event> events = Lists.newArrayList();
        for (InotifyProtos.EventProto p : bp.getEventsList()) {
            switch(p.getType()) {
                case EVENT_CLOSE:
                    InotifyProtos.CloseEventProto close = InotifyProtos.CloseEventProto.parseFrom(p.getContents());
                    events.add(new Event.CloseEvent(close.getPath(), close.getFileSize(), close.getTimestamp()));
                    break;
                case EVENT_CREATE:
                    InotifyProtos.CreateEventProto create = InotifyProtos.CreateEventProto.parseFrom(p.getContents());
                    events.add(new Event.CreateEvent.Builder().iNodeType(createTypeConvert(create.getType())).path(create.getPath()).ctime(create.getCtime()).ownerName(create.getOwnerName()).groupName(create.getGroupName()).perms(convert(create.getPerms())).replication(create.getReplication()).symlinkTarget(create.getSymlinkTarget().isEmpty() ? null : create.getSymlinkTarget()).defaultBlockSize(create.getDefaultBlockSize()).overwrite(create.getOverwrite()).build());
                    break;
                case EVENT_METADATA:
                    InotifyProtos.MetadataUpdateEventProto meta = InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
                    events.add(new Event.MetadataUpdateEvent.Builder().path(meta.getPath()).metadataType(metadataUpdateTypeConvert(meta.getType())).mtime(meta.getMtime()).atime(meta.getAtime()).replication(meta.getReplication()).ownerName(meta.getOwnerName().isEmpty() ? null : meta.getOwnerName()).groupName(meta.getGroupName().isEmpty() ? null : meta.getGroupName()).perms(meta.hasPerms() ? convert(meta.getPerms()) : null).acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(meta.getAclsList())).xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(meta.getXAttrsList())).xAttrsRemoved(meta.getXAttrsRemoved()).build());
                    break;
                case EVENT_RENAME:
                    InotifyProtos.RenameEventProto rename = InotifyProtos.RenameEventProto.parseFrom(p.getContents());
                    events.add(new Event.RenameEvent.Builder().srcPath(rename.getSrcPath()).dstPath(rename.getDestPath()).timestamp(rename.getTimestamp()).build());
                    break;
                case EVENT_APPEND:
                    InotifyProtos.AppendEventProto append = InotifyProtos.AppendEventProto.parseFrom(p.getContents());
                    events.add(new Event.AppendEvent.Builder().path(append.getPath()).newBlock(append.hasNewBlock() && append.getNewBlock()).build());
                    break;
                case EVENT_UNLINK:
                    InotifyProtos.UnlinkEventProto unlink = InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
                    events.add(new Event.UnlinkEvent.Builder().path(unlink.getPath()).timestamp(unlink.getTimestamp()).build());
                    break;
                case EVENT_TRUNCATE:
                    InotifyProtos.TruncateEventProto truncate = InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
                    events.add(new Event.TruncateEvent(truncate.getPath(), truncate.getFileSize(), truncate.getTimestamp()));
                    break;
                default:
                    throw new RuntimeException("Unexpected inotify event type: " + p.getType());
            }
        }
        batches.add(new EventBatch(txid, events.toArray(new Event[0])));
    }
    return new EventBatchList(batches, resp.getEventsList().getFirstTxid(), resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}

Example 15 with Event

use of org.apache.hadoop.hdfs.inotify.Event in project SSM by Intel-bigdata.

the class EventBatchSerializer method serialize.

//Code copy from PBHelperClient.java
public static byte[] serialize(EventBatch eventBatch) {
    List<InotifyProtos.EventProto> events = Lists.newArrayList();
    for (Event e : eventBatch.getEvents()) {
        switch(e.getEventType()) {
            case CLOSE:
                Event.CloseEvent ce = (Event.CloseEvent) e;
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_CLOSE).setContents(InotifyProtos.CloseEventProto.newBuilder().setPath(ce.getPath()).setFileSize(ce.getFileSize()).setTimestamp(ce.getTimestamp()).build().toByteString()).build());
                break;
            case CREATE:
                Event.CreateEvent ce2 = (Event.CreateEvent) e;
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_CREATE).setContents(InotifyProtos.CreateEventProto.newBuilder().setType(createTypeConvert(ce2.getiNodeType())).setPath(ce2.getPath()).setCtime(ce2.getCtime()).setOwnerName(ce2.getOwnerName()).setGroupName(ce2.getGroupName()).setPerms(convert(ce2.getPerms())).setReplication(ce2.getReplication()).setSymlinkTarget(ce2.getSymlinkTarget() == null ? "" : ce2.getSymlinkTarget()).setDefaultBlockSize(ce2.getDefaultBlockSize()).setOverwrite(ce2.getOverwrite()).build().toByteString()).build());
                break;
            case METADATA:
                Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
                InotifyProtos.MetadataUpdateEventProto.Builder metaB = InotifyProtos.MetadataUpdateEventProto.newBuilder().setPath(me.getPath()).setType(metadataUpdateTypeConvert(me.getMetadataType())).setMtime(me.getMtime()).setAtime(me.getAtime()).setReplication(me.getReplication()).setOwnerName(me.getOwnerName() == null ? "" : me.getOwnerName()).setGroupName(me.getGroupName() == null ? "" : me.getGroupName()).addAllAcls(me.getAcls() == null ? Lists.<AclProtos.AclEntryProto>newArrayList() : convertAclEntryProto(me.getAcls())).addAllXAttrs(me.getxAttrs() == null ? Lists.<XAttrProtos.XAttrProto>newArrayList() : convertXAttrProto(me.getxAttrs())).setXAttrsRemoved(me.isxAttrsRemoved());
                if (me.getPerms() != null) {
                    metaB.setPerms(convert(me.getPerms()));
                }
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_METADATA).setContents(metaB.build().toByteString()).build());
                break;
            case RENAME:
                Event.RenameEvent re = (Event.RenameEvent) e;
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_RENAME).setContents(InotifyProtos.RenameEventProto.newBuilder().setSrcPath(re.getSrcPath()).setDestPath(re.getDstPath()).setTimestamp(re.getTimestamp()).build().toByteString()).build());
                break;
            case APPEND:
                Event.AppendEvent re2 = (Event.AppendEvent) e;
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_APPEND).setContents(InotifyProtos.AppendEventProto.newBuilder().setPath(re2.getPath()).setNewBlock(re2.toNewBlock()).build().toByteString()).build());
                break;
            case UNLINK:
                Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
                events.add(InotifyProtos.EventProto.newBuilder().setType(InotifyProtos.EventType.EVENT_UNLINK).setContents(InotifyProtos.UnlinkEventProto.newBuilder().setPath(ue.getPath()).setTimestamp(ue.getTimestamp()).build().toByteString()).build());
                break;
            /*
        case TRUNCATE:
          Event.TruncateEvent te = (Event.TruncateEvent) e;
          events.add(InotifyProtos.EventProto.newBuilder()
            .setType(InotifyProtos.EventType.EVENT_TRUNCATE)
            .setContents(
              InotifyProtos.TruncateEventProto.newBuilder()
                .setPath(te.getPath())
                .setFileSize(te.getFileSize())
                .setTimestamp(te.getTimestamp()).build().toByteString()
            ).build());
          break;
          */
            default:
                throw new RuntimeException("Unexpected inotify event: " + e);
        }
    }
    return InotifyProtos.EventBatchProto.newBuilder().setTxid(eventBatch.getTxid()).addAllEvents(events).build().toByteArray();
}