Examples with EventIterator org.apache.nifi.provenance.store.iterator.EventIterator used on opensource projects

Example 1 with EventIterator

use of org.apache.nifi.provenance.store.iterator.EventIterator in project nifi by apache.

the class PartitionedEventStore method getEvents.

private List<ProvenanceEventRecord> getEvents(final int maxRecords, final EventAuthorizer authorizer, final Function<EventStorePartition, EventIterator> eventIteratorFactory, final EventTransformer transformer) throws IOException {
    if (maxRecords < 1) {
        return Collections.emptyList();
    }
    final List<ProvenanceEventRecord> selectedEvents = new ArrayList<>();
    // Create a Map so that the key is the next record available from a partition and the value is the EventIterator from which
    // the record came. This sorted map is then used so that we are able to always get the first entry, which is the next
    // lowest record id among all partitions.
    final SortedMap<ProvenanceEventRecord, EventIterator> recordToIteratorMap = new TreeMap<>((o1, o2) -> Long.compare(o1.getEventId(), o2.getEventId()));
    try {
        // Seed our map with the first event in each Partition.
        for (final EventStorePartition partition : getPartitions()) {
            final EventAuthorizer nonNullAuthorizer = authorizer == null ? EventAuthorizer.GRANT_ALL : authorizer;
            final EventIterator partitionIterator = eventIteratorFactory.apply(partition);
            final EventIterator iterator = new AuthorizingEventIterator(partitionIterator, nonNullAuthorizer, transformer);
            final Optional<ProvenanceEventRecord> option = iterator.nextEvent();
            if (option.isPresent()) {
                recordToIteratorMap.put(option.get(), iterator);
            }
        }
        // If no records found, just return the empty list.
        if (recordToIteratorMap.isEmpty()) {
            return selectedEvents;
        }
        // Get the event with the next-lowest ID. Add it to the list of selected events,
        // then read the next event from the same EventIterator that this event came from.
        // This ensures that our map is always populated with the next event for each
        // EventIterator, which also ensures that the first key in our map is the event
        // with the lowest ID (since all events from a given EventIterator have monotonically
        // increasing Event ID's).
        ProvenanceEventRecord nextEvent = recordToIteratorMap.firstKey();
        while (nextEvent != null && selectedEvents.size() < maxRecords) {
            selectedEvents.add(nextEvent);
            final EventIterator iterator = recordToIteratorMap.remove(nextEvent);
            final Optional<ProvenanceEventRecord> nextRecordFromIterator = iterator.nextEvent();
            if (nextRecordFromIterator.isPresent()) {
                recordToIteratorMap.put(nextRecordFromIterator.get(), iterator);
            }
            nextEvent = recordToIteratorMap.isEmpty() ? null : recordToIteratorMap.firstKey();
        }
        return selectedEvents;
    } finally {
        // Ensure that we close all record readers that have been created
        for (final EventIterator iterator : recordToIteratorMap.values()) {
            try {
                iterator.close();
            } catch (final Exception e) {
                if (logger.isDebugEnabled()) {
                    logger.warn("Failed to close Record Reader {}", iterator, e);
                } else {
                    logger.warn("Failed to close Record Reader {}", iterator);
                }
            }
        }
    }
}