Examples with EventBatch org.apache.hadoop.hdfs.inotify.EventBatch used on opensource projects

Example 6 with EventBatch

use of org.apache.hadoop.hdfs.inotify.EventBatch in project hadoop by apache.

the class NameNodeRpcServer method getEditsFromTxid.

// ClientProtocol
@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
    checkNNStartup();
    // only active
    namesystem.checkOperation(OperationCategory.READ);
    namesystem.checkSuperuserPrivilege();
    int maxEventsPerRPC = nn.getConf().getInt(DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY, DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
    FSEditLog log = namesystem.getFSImage().getEditLog();
    long syncTxid = log.getSyncTxId();
    // If we haven't synced anything yet, we can only read finalized
    // segments since we can't reliably determine which txns in in-progress
    // segments have actually been committed (e.g. written to a quorum of JNs).
    // If we have synced txns, we can definitely read up to syncTxid since
    // syncTxid is only updated after a transaction is committed to all
    // journals. (In-progress segments written by old writers are already
    // discarded for us, so if we read any in-progress segments they are
    // guaranteed to have been written by this NameNode.)
    boolean readInProgress = syncTxid > 0;
    List<EventBatch> batches = Lists.newArrayList();
    int totalEvents = 0;
    long maxSeenTxid = -1;
    long firstSeenTxid = -1;
    if (syncTxid > 0 && txid > syncTxid) {
        // we can't read past syncTxid, so there's no point in going any further
        return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
    }
    Collection<EditLogInputStream> streams = null;
    try {
        streams = log.selectInputStreams(txid, 0, null, readInProgress);
    } catch (IllegalStateException e) {
        // can happen if we have
        // transitioned out of active and haven't yet transitioned to standby
        // and are using QJM -- the edit log will be closed and this exception
        // will result
        LOG.info("NN is transitioning from active to standby and FSEditLog " + "is closed -- could not read edits");
        return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
    }
    boolean breakOuter = false;
    for (EditLogInputStream elis : streams) {
        // starting txid
        try {
            FSEditLogOp op = null;
            while ((op = readOp(elis)) != null) {
                // txid we get is greater than syncTxid
                if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
                    breakOuter = true;
                    break;
                }
                EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
                if (eventBatch != null) {
                    batches.add(eventBatch);
                    totalEvents += eventBatch.getEvents().length;
                }
                if (op.getTransactionId() > maxSeenTxid) {
                    maxSeenTxid = op.getTransactionId();
                }
                if (firstSeenTxid == -1) {
                    firstSeenTxid = op.getTransactionId();
                }
                if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 && op.getTransactionId() == syncTxid)) {
                    // we're done
                    breakOuter = true;
                    break;
                }
            }
        } finally {
            elis.close();
        }
        if (breakOuter) {
            break;
        }
    }
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}

Example 7 with EventBatch

use of org.apache.hadoop.hdfs.inotify.EventBatch in project nifi by apache.

the class TestGetHDFSEvents method makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch.

@Test
public void makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch() throws Exception {
    Event[] events = new Event[] { new Event.CreateEvent.Builder().path("/some/path/1/2/3/t.txt").build(), new Event.CreateEvent.Builder().path("/some/path/1/2/4/t.txt").build(), new Event.CreateEvent.Builder().path("/some/path/1/2/3/.t.txt").build() };
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);
    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);
    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/${literal(1)}/${literal(2)}/${literal(3)}/.*.txt");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create");
    runner.setProperty(GetHDFSEvents.IGNORE_HIDDEN_FILES, "true");
    runner.run();
    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());
    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(eventType.equals("CREATE"));
    }
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}

Example 8 with EventBatch

use of org.apache.hadoop.hdfs.inotify.EventBatch in project nifi by apache.

the class TestGetHDFSEvents method eventsProcessorShouldProperlyFilterEventTypes.

@Test
public void eventsProcessorShouldProperlyFilterEventTypes() throws Exception {
    Event[] events = getEvents();
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);
    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);
    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path(/.*)?");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create, metadata");
    runner.run();
    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(2, successfulFlowFiles.size());
    List<String> expectedEventTypes = Arrays.asList("CREATE", "METADATA");
    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(expectedEventTypes.contains(eventType));
    }
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}

Example 9 with EventBatch

use of org.apache.hadoop.hdfs.inotify.EventBatch in project nifi by apache.

the class TestGetHDFSEvents method onTriggerShouldProperlyHandleAnEmptyEventBatch.

@Test
public void onTriggerShouldProperlyHandleAnEmptyEventBatch() throws Exception {
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(new Event[] {});
    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);
    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);
    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path");
    runner.setProperty(GetHDFSEvents.NUMBER_OF_RETRIES_FOR_POLL, "5");
    runner.run();
    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}

Example 10 with EventBatch

use of org.apache.hadoop.hdfs.inotify.EventBatch in project SSM by Intel-bigdata.

the class InotifyFetchAndApplyTask method run.

@Override
public void run() {
    try {
        EventBatch eventBatch = inotifyEventInputStream.poll();
        while (eventBatch != null) {
            this.applier.apply(eventBatch.getEvents());
            this.lastId.getAndSet(eventBatch.getTxid());
            eventBatch = inotifyEventInputStream.poll();
        }
    } catch (IOException | MissingEventsException | SQLException e) {
        e.printStackTrace();
    }
}