Search in sources :

Example 6 with QueryResult

use of org.apache.nifi.provenance.search.QueryResult in project nifi by apache.

the class TestPersistentProvenanceRepository method testIndexDirectoryRemoved.

@Test
@Ignore("This test relies too much on timing of background events by using Thread.sleep().")
public void testIndexDirectoryRemoved() throws InterruptedException, IOException, ParseException {
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    // force new index to be created for each rollover
    config.setDesiredIndexSize(10);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        // make sure the events are destroyed when we call purge
        builder.setEventTime(10L);
        repo.registerEvent(builder.build());
    }
    repo.waitForRollover();
    Thread.sleep(2000L);
    final FileFilter indexFileFilter = file -> file.getName().startsWith("index");
    final int numIndexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter).length;
    assertEquals(1, numIndexDirs);
    // add more records so that we will create a new index
    final long secondBatchStartTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000001" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        builder.setEventTime(System.currentTimeMillis());
        repo.registerEvent(builder.build());
    }
    // wait for indexing to happen
    repo.waitForRollover();
    // verify we get the results expected
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(20, result.getMatchingEvents().size());
    // Ensure index directories exists
    File[] indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(2, indexDirs.length);
    // expire old events and indexes
    final long timeSinceSecondBatch = System.currentTimeMillis() - secondBatchStartTime;
    config.setMaxRecordLife(timeSinceSecondBatch + 1000L, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    Thread.sleep(2000L);
    final QueryResult newRecordSet = repo.queryEvents(query, createUser());
    assertEquals(10, newRecordSet.getMatchingEvents().size());
    // Ensure that one index directory is gone
    indexDirs = config.getStorageDirectories().values().iterator().next().listFiles(indexFileFilter);
    assertEquals(1, indexDirs.length);
}
Also used : Arrays(java.util.Arrays) ScoreDoc(org.apache.lucene.search.ScoreDoc) Query(org.apache.nifi.provenance.search.Query) LoggerFactory(org.slf4j.LoggerFactory) Document(org.apache.lucene.document.Document) Future(java.util.concurrent.Future) DataOutputStream(java.io.DataOutputStream) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) After(org.junit.After) Map(java.util.Map) LineageNode(org.apache.nifi.provenance.lineage.LineageNode) EventIndexWriter(org.apache.nifi.provenance.index.EventIndexWriter) ClassRule(org.junit.ClassRule) SearchableField(org.apache.nifi.provenance.search.SearchableField) Method(java.lang.reflect.Method) EventIndexSearcher(org.apache.nifi.provenance.index.EventIndexSearcher) TestUtil.createFlowFile(org.apache.nifi.provenance.TestUtil.createFlowFile) FlowFile(org.apache.nifi.flowfile.FlowFile) FileUtils(org.apache.nifi.util.file.FileUtils) DirectoryReader(org.apache.lucene.index.DirectoryReader) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Lineage(org.apache.nifi.provenance.lineage.Lineage) Executors(java.util.concurrent.Executors) InvocationTargetException(java.lang.reflect.InvocationTargetException) CountDownLatch(java.util.concurrent.CountDownLatch) IndexWriter(org.apache.lucene.index.IndexWriter) List(java.util.List) EventNode(org.apache.nifi.provenance.lineage.EventNode) IndexingAction(org.apache.nifi.provenance.lucene.IndexingAction) IndexManager(org.apache.nifi.provenance.lucene.IndexManager) GZIPOutputStream(java.util.zip.GZIPOutputStream) SearchTerms(org.apache.nifi.provenance.search.SearchTerms) IndexSearcher(org.apache.lucene.search.IndexSearcher) Mockito.mock(org.mockito.Mockito.mock) ParseException(org.apache.lucene.queryparser.classic.ParseException) BeforeClass(org.junit.BeforeClass) Assume.assumeFalse(org.junit.Assume.assumeFalse) AccessDeniedException(org.apache.nifi.authorization.AccessDeniedException) RecordWriter(org.apache.nifi.provenance.serialization.RecordWriter) HashMap(java.util.HashMap) RecordReaders(org.apache.nifi.provenance.serialization.RecordReaders) CachingIndexManager(org.apache.nifi.provenance.lucene.CachingIndexManager) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) QueryResult(org.apache.nifi.provenance.search.QueryResult) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) TestName(org.junit.rules.TestName) LineageEdge(org.apache.nifi.provenance.lineage.LineageEdge) FSDirectory(org.apache.lucene.store.FSDirectory) QuerySubmission(org.apache.nifi.provenance.search.QuerySubmission) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) TopDocs(org.apache.lucene.search.TopDocs) Logger(org.slf4j.Logger) Assert.assertNotNull(org.junit.Assert.assertNotNull) Analyzer(org.apache.lucene.analysis.Analyzer) SimpleAnalyzer(org.apache.lucene.analysis.core.SimpleAnalyzer) FileOutputStream(java.io.FileOutputStream) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) Field(java.lang.reflect.Field) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecordWriters(org.apache.nifi.provenance.serialization.RecordWriters) Rule(org.junit.Rule) FileFilter(java.io.FileFilter) Assert.assertNull(org.junit.Assert.assertNull) EventReporter(org.apache.nifi.events.EventReporter) NiFiProperties(org.apache.nifi.util.NiFiProperties) Ignore(org.junit.Ignore) QueryParser(org.apache.lucene.queryparser.classic.QueryParser) Severity(org.apache.nifi.reporting.Severity) RecordReader(org.apache.nifi.provenance.serialization.RecordReader) Assert(org.junit.Assert) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) LineageNodeType(org.apache.nifi.provenance.lineage.LineageNodeType) Query(org.apache.nifi.provenance.search.Query) HashMap(java.util.HashMap) QueryResult(org.apache.nifi.provenance.search.QueryResult) FileFilter(java.io.FileFilter) TestUtil.createFlowFile(org.apache.nifi.provenance.TestUtil.createFlowFile) FlowFile(org.apache.nifi.flowfile.FlowFile) File(java.io.File) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 7 with QueryResult

use of org.apache.nifi.provenance.search.QueryResult in project nifi by apache.

the class TestPersistentProvenanceRepository method testIndexOnRolloverWithImmenseAttribute.

@Test
public void testIndexOnRolloverWithImmenseAttribute() throws IOException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    config.setSearchableAttributes(SearchableFieldParser.extractSearchableFields("immense", false));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    // must be greater than 32766 for a meaningful test
    int immenseAttrSize = 33000;
    StringBuilder immenseBldr = new StringBuilder(immenseAttrSize);
    for (int i = 0; i < immenseAttrSize; i++) {
        immenseBldr.append('0');
    }
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    attributes.put("immense", immenseBldr.toString());
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }
    repo.waitForRollover();
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.newSearchableAttribute("immense"), "000*"));
    query.setMaxResults(100);
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
}
Also used : QueryResult(org.apache.nifi.provenance.search.QueryResult) Query(org.apache.nifi.provenance.search.Query) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 8 with QueryResult

use of org.apache.nifi.provenance.search.QueryResult in project nifi by apache.

the class TestPersistentProvenanceRepository method testIndexOnRolloverAndSubsequentSearch.

@Test
public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }
    repo.waitForRollover();
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);
    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(10, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
Also used : QueryResult(org.apache.nifi.provenance.search.QueryResult) Query(org.apache.nifi.provenance.search.Query) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 9 with QueryResult

use of org.apache.nifi.provenance.search.QueryResult in project nifi by apache.

the class TestPersistentProvenanceRepository method testNotAuthorizedQuery.

@Test(timeout = 10000)
public void testNotAuthorizedQuery() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(5, TimeUnit.MINUTES);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    // force new index to be created for each rollover
    config.setDesiredIndexSize(10);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) {
            return event.getEventId() > 2;
        }
    };
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        // make sure the events are destroyed when we call purge
        builder.setEventTime(10L);
        repo.registerEvent(builder.build());
    }
    repo.waitForRollover();
    final Query query = new Query("1234");
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234"));
    final QuerySubmission submission = repo.submitQuery(query, createUser());
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        Thread.sleep(100L);
    }
    // Ensure that we gets events with ID's 3 through 10.
    final List<ProvenanceEventRecord> events = result.getMatchingEvents();
    assertEquals(7, events.size());
    final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList());
    for (int i = 0; i < 7; i++) {
        Assert.assertEquals(i + 3, eventIds.get(i).intValue());
    }
}
Also used : QuerySubmission(org.apache.nifi.provenance.search.QuerySubmission) NiFiUser(org.apache.nifi.authorization.user.NiFiUser) Query(org.apache.nifi.provenance.search.Query) HashMap(java.util.HashMap) QueryResult(org.apache.nifi.provenance.search.QueryResult) AtomicLong(java.util.concurrent.atomic.AtomicLong) Test(org.junit.Test)

Example 10 with QueryResult

use of org.apache.nifi.provenance.search.QueryResult in project nifi by apache.

the class PersistentProvenanceRepository method queryEvents.

QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
    }
    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    logger.info("{} got {} hits", query, result.getTotalHitCount());
    return result;
}
Also used : QueryResult(org.apache.nifi.provenance.search.QueryResult) QuerySubmission(org.apache.nifi.provenance.search.QuerySubmission) IOException(java.io.IOException)

Aggregations

QueryResult (org.apache.nifi.provenance.search.QueryResult)16 Query (org.apache.nifi.provenance.search.Query)13 Test (org.junit.Test)12 HashMap (java.util.HashMap)9 QuerySubmission (org.apache.nifi.provenance.search.QuerySubmission)8 IOException (java.io.IOException)6 File (java.io.File)4 FlowFile (org.apache.nifi.flowfile.FlowFile)4 TestUtil.createFlowFile (org.apache.nifi.provenance.TestUtil.createFlowFile)4 IndexManager (org.apache.nifi.provenance.lucene.IndexManager)4 DataOutputStream (java.io.DataOutputStream)3 FileOutputStream (java.io.FileOutputStream)3 ArrayList (java.util.ArrayList)3 GZIPOutputStream (java.util.zip.GZIPOutputStream)3 HashSet (java.util.HashSet)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)2 NiFiUser (org.apache.nifi.authorization.user.NiFiUser)2 ProvenanceEventRecord (org.apache.nifi.provenance.ProvenanceEventRecord)2