Search in sources :

Example 1 with SimpleIndexManager

use of org.apache.nifi.provenance.lucene.SimpleIndexManager in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetFilteredForQuery.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetFilteredForQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final Query query = new Query(UUID.randomUUID().toString());
    final EventAuthorizer authorizer = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventId() % 2 == 0;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
            throw new AccessDeniedException();
        }
    };
    List<ProvenanceEventRecord> events = Collections.emptyList();
    while (events.size() < 2) {
        final QuerySubmission submission = index.submitQuery(query, authorizer, "unit test");
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        events = submission.getResult().getMatchingEvents();
        Thread.sleep(25L);
    }
    assertEquals(2, events.size());
}
Also used : AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) QuerySubmission(org.apache.nifi.provenance.search.QuerySubmission) Query(org.apache.nifi.provenance.search.Query) EventAuthorizer(org.apache.nifi.provenance.authorization.EventAuthorizer) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) ArrayListEventStore(org.apache.nifi.provenance.store.ArrayListEventStore) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) RepositoryConfiguration(org.apache.nifi.provenance.RepositoryConfiguration) StorageResult(org.apache.nifi.provenance.store.StorageResult) Test(org.junit.Test)

Example 2 with SimpleIndexManager

use of org.apache.nifi.provenance.lucene.SimpleIndexManager in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForFindParents.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForFindParents() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    final ProvenanceEventRecord firstEvent = createEvent("4444");
    final Map<String, String> previousAttributes = new HashMap<>();
    previousAttributes.put("uuid", "4444");
    final Map<String, String> updatedAttributes = new HashMap<>();
    updatedAttributes.put("updated", "true");
    final ProvenanceEventRecord join = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.JOIN).setAttributes(previousAttributes, updatedAttributes).addParentUuid("4444").addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("1234").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
    index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
    index.addEvents(eventStore.addEvent(join).getStorageLocations());
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    final EventAuthorizer allowJoinEvents = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventType() == ProvenanceEventType.JOIN;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
        }
    };
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 2) {
        final ComputeLineageSubmission submission = index.submitExpandParents(1L, user, allowJoinEvents);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(2, nodes.size());
    final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
    assertEquals(2, eventMap.size());
    assertEquals(1, eventMap.get(ProvenanceEventType.JOIN).size());
    assertEquals(1, eventMap.get(ProvenanceEventType.UNKNOWN).size());
    assertEquals("4444", eventMap.get(ProvenanceEventType.UNKNOWN).get(0).getFlowFileUuid());
}
Also used : BeforeClass(org.junit.BeforeClass) Query(org.apache.nifi.provenance.search.Query) Assume.assumeFalse(org.junit.Assume.assumeFalse) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) SearchableFields(org.apache.nifi.provenance.SearchableFields) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Answer(org.mockito.stubbing.Answer) InvocationOnMock(org.mockito.invocation.InvocationOnMock) QueryResult(org.apache.nifi.provenance.search.QueryResult) StorageResult(org.apache.nifi.provenance.store.StorageResult) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) TestName(org.junit.rules.TestName) Map(java.util.Map) EventStore(org.apache.nifi.provenance.store.EventStore) LineageNode(org.apache.nifi.provenance.lineage.LineageNode) QuerySubmission(org.apache.nifi.provenance.search.QuerySubmission) StorageSummary(org.apache.nifi.provenance.serialization.StorageSummary) ProvenanceEventType(org.apache.nifi.provenance.ProvenanceEventType) Assert.assertNotNull(org.junit.Assert.assertNotNull) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) Rule(org.junit.Rule) Assert.assertNull(org.junit.Assert.assertNull) EventReporter(org.apache.nifi.events.EventReporter) ComputeLineageSubmission(org.apache.nifi.provenance.lineage.ComputeLineageSubmission) RepositoryConfiguration(org.apache.nifi.provenance.RepositoryConfiguration) ProvenanceEventLineageNode(org.apache.nifi.provenance.lineage.ProvenanceEventLineageNode) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) ArrayListEventStore(org.apache.nifi.provenance.store.ArrayListEventStore) Collections(java.util.Collections) EventAuthorizer(org.apache.nifi.provenance.authorization.EventAuthorizer) SearchTerms(org.apache.nifi.provenance.search.SearchTerms) Assert.assertEquals(org.junit.Assert.assertEquals) LineageNodeType(org.apache.nifi.provenance.lineage.LineageNodeType) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) HashMap(java.util.HashMap) EventAuthorizer(org.apache.nifi.provenance.authorization.EventAuthorizer) ComputeLineageSubmission(org.apache.nifi.provenance.lineage.ComputeLineageSubmission) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) ArrayListEventStore(org.apache.nifi.provenance.store.ArrayListEventStore) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) ProvenanceEventLineageNode(org.apache.nifi.provenance.lineage.ProvenanceEventLineageNode) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) ArrayList(java.util.ArrayList) List(java.util.List) LineageNode(org.apache.nifi.provenance.lineage.LineageNode) ProvenanceEventLineageNode(org.apache.nifi.provenance.lineage.ProvenanceEventLineageNode) RepositoryConfiguration(org.apache.nifi.provenance.RepositoryConfiguration) ProvenanceEventType(org.apache.nifi.provenance.ProvenanceEventType) StorageResult(org.apache.nifi.provenance.store.StorageResult) Test(org.junit.Test)

Example 3 with SimpleIndexManager

use of org.apache.nifi.provenance.lucene.SimpleIndexManager in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForLineage.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 3) {
        final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(3, nodes.size());
    for (final LineageNode node : nodes) {
        assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
        final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
        assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
    }
}
Also used : NiFiUser(org.apache.nifi.authorization.user.NiFiUser) ComputeLineageSubmission(org.apache.nifi.provenance.lineage.ComputeLineageSubmission) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) ArrayListEventStore(org.apache.nifi.provenance.store.ArrayListEventStore) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) ProvenanceEventLineageNode(org.apache.nifi.provenance.lineage.ProvenanceEventLineageNode) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) LineageNode(org.apache.nifi.provenance.lineage.LineageNode) ProvenanceEventLineageNode(org.apache.nifi.provenance.lineage.ProvenanceEventLineageNode) RepositoryConfiguration(org.apache.nifi.provenance.RepositoryConfiguration) StorageResult(org.apache.nifi.provenance.store.StorageResult) Test(org.junit.Test)

Example 4 with SimpleIndexManager

use of org.apache.nifi.provenance.lucene.SimpleIndexManager in project nifi by apache.

the class WriteAheadProvenanceRepository method init.

synchronized void init(RecordWriterFactory recordWriterFactory, RecordReaderFactory recordReaderFactory, final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory) throws IOException {
    final EventFileManager fileManager = new EventFileManager();
    eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);
    final IndexManager indexManager = new SimpleIndexManager(config);
    eventIndex = new LuceneEventIndex(config, indexManager, eventReporter);
    this.eventReporter = eventReporter;
    this.authorizer = authorizer;
    this.resourceFactory = resourceFactory;
    eventStore.initialize();
    eventIndex.initialize(eventStore);
    try {
        eventStore.reindexLatestEvents(eventIndex);
    } catch (final Exception e) {
        logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest " + "events will not be available from the Provenance Repository when a query is issued.", e);
    }
}
Also used : SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) PartitionedWriteAheadEventStore(org.apache.nifi.provenance.store.PartitionedWriteAheadEventStore) EventFileManager(org.apache.nifi.provenance.store.EventFileManager) LuceneEventIndex(org.apache.nifi.provenance.index.lucene.LuceneEventIndex) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IOException(java.io.IOException)

Example 5 with SimpleIndexManager

use of org.apache.nifi.provenance.lucene.SimpleIndexManager in project nifi by apache.

the class TestLuceneEventIndex method testGetMinimumIdToReindex.

@Test(timeout = 60000)
public void testGetMinimumIdToReindex() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 20_000, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 50_000; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    while (index.getMaxEventId("1") < 40_000L) {
        Thread.sleep(25);
    }
    final long id = index.getMinimumEventIdToReindex("1");
    assertTrue(id >= 30000L);
}
Also used : SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) StandardProvenanceEventRecord(org.apache.nifi.provenance.StandardProvenanceEventRecord) RepositoryConfiguration(org.apache.nifi.provenance.RepositoryConfiguration) SimpleIndexManager(org.apache.nifi.provenance.lucene.SimpleIndexManager) ArrayListEventStore(org.apache.nifi.provenance.store.ArrayListEventStore) StorageResult(org.apache.nifi.provenance.store.StorageResult) Test(org.junit.Test)

Aggregations

IndexManager (org.apache.nifi.provenance.lucene.IndexManager)9 SimpleIndexManager (org.apache.nifi.provenance.lucene.SimpleIndexManager)9 ProvenanceEventRecord (org.apache.nifi.provenance.ProvenanceEventRecord)8 RepositoryConfiguration (org.apache.nifi.provenance.RepositoryConfiguration)8 StandardProvenanceEventRecord (org.apache.nifi.provenance.StandardProvenanceEventRecord)8 ArrayListEventStore (org.apache.nifi.provenance.store.ArrayListEventStore)8 Test (org.junit.Test)8 Query (org.apache.nifi.provenance.search.Query)5 QuerySubmission (org.apache.nifi.provenance.search.QuerySubmission)5 StorageResult (org.apache.nifi.provenance.store.StorageResult)5 QueryResult (org.apache.nifi.provenance.search.QueryResult)4 StorageSummary (org.apache.nifi.provenance.serialization.StorageSummary)4 File (java.io.File)3 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3 AtomicLong (java.util.concurrent.atomic.AtomicLong)3 AccessDeniedException (org.apache.nifi.authorization.AccessDeniedException)3 NiFiUser (org.apache.nifi.authorization.user.NiFiUser)3 EventAuthorizer (org.apache.nifi.provenance.authorization.EventAuthorizer)3