use of org.apache.nifi.provenance.search.Query in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetFilteredForQuery.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetFilteredForQuery() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final Query query = new Query(UUID.randomUUID().toString());
final EventAuthorizer authorizer = new EventAuthorizer() {
@Override
public boolean isAuthorized(ProvenanceEventRecord event) {
return event.getEventId() % 2 == 0;
}
@Override
public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
throw new AccessDeniedException();
}
};
List<ProvenanceEventRecord> events = Collections.emptyList();
while (events.size() < 2) {
final QuerySubmission submission = index.submitQuery(query, authorizer, "unit test");
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
events = submission.getResult().getMatchingEvents();
Thread.sleep(25L);
}
assertEquals(2, events.size());
}
use of org.apache.nifi.provenance.search.Query in project nifi by apache.
the class TestVolatileProvenanceRepository method testIndexAndCompressOnRolloverAndSubsequentSearchAsync.
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null));
final String uuid = "00000000-0000-0000-0000-000000000000";
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
attributes.put("xyz", "abc");
attributes.put("filename", "file-" + uuid);
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
for (int i = 0; i < 10; i++) {
attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
repo.registerEvent(builder.build());
}
final Query query = new Query(UUID.randomUUID().toString());
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
query.setMaxResults(100);
final QuerySubmission submission = repo.submitQuery(query, createUser());
while (!submission.getResult().isFinished()) {
Thread.sleep(100L);
}
assertEquals(10, submission.getResult().getMatchingEvents().size());
for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
System.out.println(match);
}
}
use of org.apache.nifi.provenance.search.Query in project nifi by apache.
the class TestPersistentProvenanceRepository method runQuery.
private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
assumeFalse(isWindowsEnvironment());
try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory))) {
final IndexSearcher searcher = new IndexSearcher(directoryReader);
final Analyzer analyzer = new SimpleAnalyzer();
final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);
final Query q = new Query("");
q.setMaxResults(1000);
final TopDocs topDocs = searcher.search(luceneQuery, 1000);
final List<Document> docs = new ArrayList<>();
for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
final int docId = scoreDoc.doc;
final Document d = directoryReader.document(docId);
docs.add(d);
}
return docs;
}
}
use of org.apache.nifi.provenance.search.Query in project nifi by apache.
the class TestPersistentProvenanceRepository method testAddToMultipleLogsAndRecover.
@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
assumeFalse(isWindowsEnvironment());
final List<SearchableField> searchableFields = new ArrayList<>();
searchableFields.add(SearchableFields.ComponentID);
final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileCapacity(1024L * 1024L);
config.setMaxEventFileLife(2, TimeUnit.SECONDS);
config.setSearchableFields(searchableFields);
repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
attributes.put("xyz", "abc");
attributes.put("uuid", UUID.randomUUID().toString());
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
final ProvenanceEventRecord record = builder.build();
for (int i = 0; i < 10; i++) {
repo.registerEvent(record);
}
// create a different component id so that we can make sure we query this record.
builder.setComponentId("XXXX");
attributes.put("uuid", "11111111-1111-1111-1111-111111111111");
builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
repo.registerEvent(builder.build());
repo.waitForRollover();
// Give the repo time to shutdown (i.e., close all file handles, etc.)
Thread.sleep(500L);
// Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
// log file than the previous one.
attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
repo.registerEvent(builder.build());
repo.waitForRollover();
final Query query = new Query(UUID.randomUUID().toString());
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
query.setMaxResults(100);
final QueryResult result = repo.queryEvents(query, createUser());
assertEquals(2, result.getMatchingEvents().size());
for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
System.out.println(match);
}
}
use of org.apache.nifi.provenance.search.Query in project nifi by apache.
the class TestPersistentProvenanceRepository method testIndexAndCompressOnRolloverAndSubsequentSearch.
@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration config = createConfiguration();
config.setMaxRecordLife(30, TimeUnit.SECONDS);
config.setMaxStorageCapacity(1024L * 1024L * 10);
config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
config.setMaxEventFileCapacity(1024L * 1024L * 10);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
final String uuid = "10000000-0000-0000-0000-000000000000";
final Map<String, String> attributes = new HashMap<>();
attributes.put("abc", "xyz");
attributes.put("xyz", "abc");
attributes.put("filename", "file-" + uuid);
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
attributes.put("uuid", uuid);
builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
for (int i = 0; i < 10; i++) {
builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
repo.registerEvent(builder.build());
}
repo.waitForRollover();
final Query query = new Query(UUID.randomUUID().toString());
// query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
query.setMaxResults(100);
final QueryResult result = repo.queryEvents(query, createUser());
assertEquals(10, result.getMatchingEvents().size());
for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
System.out.println(match);
}
Thread.sleep(2000L);
config.setMaxStorageCapacity(100L);
config.setMaxRecordLife(500, TimeUnit.MILLISECONDS);
repo.purgeOldEvents();
Thread.sleep(2000L);
final QueryResult newRecordSet = repo.queryEvents(query, createUser());
assertTrue(newRecordSet.getMatchingEvents().isEmpty());
}
Aggregations