Examples with QuerySubmission org.apache.nifi.provenance.search.QuerySubmission used on opensource projects

Example 6 with QuerySubmission

use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.

the class LuceneEventIndex method retrieveQuerySubmission.

@Override
public QuerySubmission retrieveQuerySubmission(final String queryIdentifier, final NiFiUser user) {
    final QuerySubmission submission = querySubmissionMap.get(queryIdentifier);
    final String userId = submission.getSubmitterIdentity();
    if (user == null && userId == null) {
        return submission;
    }
    if (user == null) {
        throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because no user id was provided");
    }
    if (userId == null || userId.equals(user.getIdentity())) {
        return submission;
    }
    throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because " + user.getIdentity() + " is not the user who submitted the request");
}

Example 7 with QuerySubmission

use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.

the class VolatileProvenanceRepository method retrieveQuerySubmission.

@Override
public QuerySubmission retrieveQuerySubmission(final String queryIdentifier, final NiFiUser user) {
    final QuerySubmission submission = querySubmissionMap.get(queryIdentifier);
    final String userId = submission.getSubmitterIdentity();
    if (user == null && userId == null) {
        return submission;
    }
    if (user == null) {
        throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because no user id was provided in the provenance request.");
    }
    if (userId == null || userId.equals(user.getIdentity())) {
        return submission;
    }
    throw new AccessDeniedException("Cannot retrieve Provenance Query Submission because " + user.getIdentity() + " is not the user who submitted the request.");
}

Example 8 with QuerySubmission

use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.

the class VolatileProvenanceRepository method queryEvents.

public QueryResult queryEvents(final Query query, final NiFiUser user) throws IOException {
    final QuerySubmission submission = submitQuery(query, user);
    final QueryResult result = submission.getResult();
    while (!result.isFinished()) {
        try {
            Thread.sleep(100L);
        } catch (final InterruptedException ie) {
        }
    }
    if (result.getError() != null) {
        throw new IOException(result.getError());
    }
    return result;
}

Example 9 with QuerySubmission

use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.

the class TestVolatileProvenanceRepository method testIndexAndCompressOnRolloverAndSubsequentSearchAsync.

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchAsync() throws InterruptedException {
    repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null));
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    for (int i = 0; i < 10; i++) {
        attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i);
        builder.fromFlowFile(createFlowFile(i, 3000L, attributes));
        repo.registerEvent(builder.build());
    }
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);
    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }
    assertEquals(10, submission.getResult().getMatchingEvents().size());
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
    }
}

Example 10 with QuerySubmission

use of org.apache.nifi.provenance.search.QuerySubmission in project nifi by apache.

the class TestPersistentProvenanceRepository method testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs.

@Test
public void testIndexAndCompressOnRolloverAndSubsequentSearchMultipleStorageDirs() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.addStorageDirectory("2", new File("target/storage/" + UUID.randomUUID().toString()));
    config.setMaxRecordLife(30, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(1, TimeUnit.SECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000000";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    for (int j = 0; j < 3; j++) {
        attributes.put("iteration", String.valueOf(j));
        builder.setEventTime(System.currentTimeMillis());
        builder.setEventType(ProvenanceEventType.RECEIVE);
        builder.setTransitUri("nifi://unit-test");
        builder.setComponentId("1234");
        builder.setComponentType("dummy processor");
        builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
        for (int i = 0; i < 10; i++) {
            String uuidSuffix = String.valueOf(i + j * 10);
            if (uuidSuffix.length() < 2) {
                uuidSuffix = "0" + uuidSuffix;
            }
            attributes.put("uuid", "00000000-0000-0000-0000-0000000000" + uuidSuffix);
            builder.fromFlowFile(createFlowFile(i + j * 10, 3000L, attributes));
            repo.registerEvent(builder.build());
        }
        repo.waitForRollover();
    }
    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4"));
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*"));
    query.setMaxResults(100);
    final QuerySubmission submission = repo.submitQuery(query, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }
    assertEquals(30, submission.getResult().getMatchingEvents().size());
    final Map<String, Integer> counts = new HashMap<>();
    for (final ProvenanceEventRecord match : submission.getResult().getMatchingEvents()) {
        System.out.println(match);
        final String index = match.getAttributes().get("iteration");
        Integer count = counts.get(index);
        if (count == null) {
            count = 0;
        }
        counts.put(index, count + 1);
    }
    assertEquals(3, counts.size());
    assertEquals(10, counts.get("0").intValue());
    assertEquals(10, counts.get("1").intValue());
    assertEquals(10, counts.get("2").intValue());
    config.setMaxRecordLife(1, TimeUnit.MILLISECONDS);
    repo.purgeOldEvents();
    // purge is async. Give it time to do its job.
    Thread.sleep(2000L);
    query.setMaxResults(100);
    final QuerySubmission noResultSubmission = repo.submitQuery(query, createUser());
    while (!noResultSubmission.getResult().isFinished()) {
        Thread.sleep(10L);
    }
    assertEquals(0, noResultSubmission.getResult().getTotalHitCount());
}