Examples with Query org.apache.nifi.provenance.search.Query used on opensource projects

Example 16 with Query

use of org.apache.nifi.provenance.search.Query in project nifi by apache.

the class TestPersistentProvenanceRepository method testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs.

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));
        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }
            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }
        repo.waitForRollover();
    }
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);
    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }
    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }
    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());
    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    // purge is async. Give it time to do its job.
    Thread.sleep(2000L);
    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }
    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}

Example 17 with Query

use of org.apache.nifi.provenance.search.Query in project nifi by apache.

the class TestLuceneEventIndex method addThenQueryWithEmptyQuery.

@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    final Query query = new Query(UUID.randomUUID().toString());
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);
    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);
        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);
        assertTrue(result.isFinished());
        assertNull(result.getError());
        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        // avoid crushing the CPU
        Thread.sleep(100L);
    }
    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}

Example 18 with Query

use of org.apache.nifi.provenance.search.Query in project nifi by apache.

the class TestLuceneEventIndex method testQuerySpecificField.

@Test(timeout = 50000)
public void testQuerySpecificField() throws InterruptedException {
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP);
    // add 2 events, one of which we will query for.
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L));
    // Create a query that searches for the event with the FlowFile UUID equal to the first event's.
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid()));
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);
    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);
        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);
        assertTrue(result.isFinished());
        assertNull(result.getError());
        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        // avoid crushing the CPU
        Thread.sleep(100L);
    }
    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}