Examples with EventAuthorizer org.apache.nifi.provenance.authorization.EventAuthorizer used on opensource projects

Example 1 with EventAuthorizer

use of org.apache.nifi.provenance.authorization.EventAuthorizer in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetFilteredForQuery.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetFilteredForQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final Query query = new Query(UUID.randomUUID().toString());
    final EventAuthorizer authorizer = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventId() % 2 == 0;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
            throw new AccessDeniedException();
        }
    };
    List<ProvenanceEventRecord> events = Collections.emptyList();
    while (events.size() < 2) {
        final QuerySubmission submission = index.submitQuery(query, authorizer, "unit test");
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        events = submission.getResult().getMatchingEvents();
        Thread.sleep(25L);
    }
    assertEquals(2, events.size());
}

Example 2 with EventAuthorizer

use of org.apache.nifi.provenance.authorization.EventAuthorizer in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForFindParents.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForFindParents() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    final ProvenanceEventRecord firstEvent = createEvent("4444");
    final Map<String, String> previousAttributes = new HashMap<>();
    previousAttributes.put("uuid", "4444");
    final Map<String, String> updatedAttributes = new HashMap<>();
    updatedAttributes.put("updated", "true");
    final ProvenanceEventRecord join = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.JOIN).setAttributes(previousAttributes, updatedAttributes).addParentUuid("4444").addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("1234").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
    index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
    index.addEvents(eventStore.addEvent(join).getStorageLocations());
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    final EventAuthorizer allowJoinEvents = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventType() == ProvenanceEventType.JOIN;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
        }
    };
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 2) {
        final ComputeLineageSubmission submission = index.submitExpandParents(1L, user, allowJoinEvents);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(2, nodes.size());
    final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
    assertEquals(2, eventMap.size());
    assertEquals(1, eventMap.get(ProvenanceEventType.JOIN).size());
    assertEquals(1, eventMap.get(ProvenanceEventType.UNKNOWN).size());
    assertEquals("4444", eventMap.get(ProvenanceEventType.UNKNOWN).get(0).getFlowFileUuid());
}

Example 3 with EventAuthorizer

use of org.apache.nifi.provenance.authorization.EventAuthorizer in project nifi by apache.

the class TestPartitionedWriteAheadEventStore method testGetEventsWithStartOffsetAndCountWithNothingAuthorized.

@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();
    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }
    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}

Example 4 with EventAuthorizer

use of org.apache.nifi.provenance.authorization.EventAuthorizer in project nifi by apache.

the class IndexSearch method search.

public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final NiFiUser user, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException {
    if (retrievedCount.get() >= provenanceQuery.getMaxResults()) {
        final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1);
        sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0L);
        logger.info("Skipping search of Provenance Index {} for {} because the max number of results ({}) has already been retrieved", indexDirectory, provenanceQuery, provenanceQuery.getMaxResults());
        return sqr;
    }
    final long startNanos = System.nanoTime();
    if (!indexDirectory.exists() && !indexDirectory.mkdirs()) {
        throw new IOException("Unable to create Indexing Directory " + indexDirectory);
    }
    if (!indexDirectory.isDirectory()) {
        throw new IOException("Indexing Directory specified is " + indexDirectory + ", but this is not a directory");
    }
    final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1);
    final Set<ProvenanceEventRecord> matchingRecords;
    // the repository, and we don't want those events to count toward the total number of matches.
    if (provenanceQuery.getStartDate() == null || provenanceQuery.getStartDate().getTime() < firstEventTimestamp) {
        provenanceQuery.setStartDate(new Date(firstEventTimestamp));
    }
    if (provenanceQuery.getEndDate() == null) {
        provenanceQuery.setEndDate(new Date());
    }
    final Query luceneQuery = LuceneUtil.convertQuery(provenanceQuery);
    final long start = System.nanoTime();
    EventIndexSearcher searcher = null;
    try {
        searcher = indexManager.borrowIndexSearcher(indexDirectory);
        final long searchStartNanos = System.nanoTime();
        final long openSearcherNanos = searchStartNanos - start;
        logger.debug("Searching {} for {}", this, provenanceQuery);
        final TopDocs topDocs = searcher.getIndexSearcher().search(luceneQuery, provenanceQuery.getMaxResults());
        final long finishSearch = System.nanoTime();
        final long searchNanos = finishSearch - searchStartNanos;
        logger.debug("Searching {} for {} took {} millis; opening searcher took {} millis", this, provenanceQuery, TimeUnit.NANOSECONDS.toMillis(searchNanos), TimeUnit.NANOSECONDS.toMillis(openSearcherNanos));
        if (topDocs.totalHits == 0) {
            sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0);
            return sqr;
        }
        final DocsReader docsReader = new DocsReader();
        final EventAuthorizer authorizer = new EventAuthorizer() {

            @Override
            public boolean isAuthorized(ProvenanceEventRecord event) {
                return repository.isAuthorized(event, user);
            }

            @Override
            public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
                repository.authorize(event, user);
            }

            @Override
            public List<ProvenanceEventRecord> filterUnauthorizedEvents(List<ProvenanceEventRecord> events) {
                return repository.filterUnauthorizedEvents(events, user);
            }

            @Override
            public Set<ProvenanceEventRecord> replaceUnauthorizedWithPlaceholders(Set<ProvenanceEventRecord> events) {
                return repository.replaceUnauthorizedWithPlaceholders(events, user);
            }
        };
        matchingRecords = docsReader.read(topDocs, authorizer, searcher.getIndexSearcher().getIndexReader(), repository.getAllLogFiles(), retrievedCount, provenanceQuery.getMaxResults(), maxAttributeChars);
        final long readRecordsNanos = System.nanoTime() - finishSearch;
        logger.debug("Reading {} records took {} millis for {}", matchingRecords.size(), TimeUnit.NANOSECONDS.toMillis(readRecordsNanos), this);
        sqr.update(matchingRecords, topDocs.totalHits);
        final long queryNanos = System.nanoTime() - startNanos;
        logger.info("Successfully executed {} against Index {}; Search took {} milliseconds; Total Hits = {}", provenanceQuery, indexDirectory, TimeUnit.NANOSECONDS.toMillis(queryNanos), topDocs.totalHits);
        return sqr;
    } catch (final FileNotFoundException e) {
        // nothing has been indexed yet, or the data has already aged off
        logger.warn("Attempted to search Provenance Index {} but could not find the file due to {}", indexDirectory, e);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
        sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0);
        return sqr;
    } finally {
        if (searcher != null) {
            indexManager.returnIndexSearcher(searcher);
        }
    }
}

Example 5 with EventAuthorizer

use of org.apache.nifi.provenance.authorization.EventAuthorizer in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForExpandChildren.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForExpandChildren() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    final ProvenanceEventRecord firstEvent = createEvent("4444");
    final Map<String, String> previousAttributes = new HashMap<>();
    previousAttributes.put("uuid", "4444");
    final Map<String, String> updatedAttributes = new HashMap<>();
    updatedAttributes.put("updated", "true");
    final ProvenanceEventRecord fork = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.FORK).setAttributes(previousAttributes, updatedAttributes).addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("4444").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
    index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
    index.addEvents(eventStore.addEvent(fork).getStorageLocations());
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    final EventAuthorizer allowForkEvents = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventType() == ProvenanceEventType.FORK;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
        }
    };
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 5) {
        final ComputeLineageSubmission submission = index.submitExpandChildren(1L, user, allowForkEvents);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(5, nodes.size());
    assertEquals(1L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.FLOWFILE_NODE).count());
    assertEquals(4L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).count());
    final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
    assertEquals(2, eventMap.size());
    assertEquals(1, eventMap.get(ProvenanceEventType.FORK).size());
    assertEquals(3, eventMap.get(ProvenanceEventType.UNKNOWN).size());
}