Examples with StandardQueryResult org.apache.nifi.provenance.StandardQueryResult used on opensource projects

Example 1 with StandardQueryResult

use of org.apache.nifi.provenance.StandardQueryResult in project nifi by apache.

the class IndexSearch method search.

public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final NiFiUser user, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException {
    if (retrievedCount.get() >= provenanceQuery.getMaxResults()) {
        final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1);
        sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0L);
        logger.info("Skipping search of Provenance Index {} for {} because the max number of results ({}) has already been retrieved", indexDirectory, provenanceQuery, provenanceQuery.getMaxResults());
        return sqr;
    }
    final long startNanos = System.nanoTime();
    if (!indexDirectory.exists() && !indexDirectory.mkdirs()) {
        throw new IOException("Unable to create Indexing Directory " + indexDirectory);
    }
    if (!indexDirectory.isDirectory()) {
        throw new IOException("Indexing Directory specified is " + indexDirectory + ", but this is not a directory");
    }
    final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1);
    final Set<ProvenanceEventRecord> matchingRecords;
    // the repository, and we don't want those events to count toward the total number of matches.
    if (provenanceQuery.getStartDate() == null || provenanceQuery.getStartDate().getTime() < firstEventTimestamp) {
        provenanceQuery.setStartDate(new Date(firstEventTimestamp));
    }
    if (provenanceQuery.getEndDate() == null) {
        provenanceQuery.setEndDate(new Date());
    }
    final Query luceneQuery = LuceneUtil.convertQuery(provenanceQuery);
    final long start = System.nanoTime();
    EventIndexSearcher searcher = null;
    try {
        searcher = indexManager.borrowIndexSearcher(indexDirectory);
        final long searchStartNanos = System.nanoTime();
        final long openSearcherNanos = searchStartNanos - start;
        logger.debug("Searching {} for {}", this, provenanceQuery);
        final TopDocs topDocs = searcher.getIndexSearcher().search(luceneQuery, provenanceQuery.getMaxResults());
        final long finishSearch = System.nanoTime();
        final long searchNanos = finishSearch - searchStartNanos;
        logger.debug("Searching {} for {} took {} millis; opening searcher took {} millis", this, provenanceQuery, TimeUnit.NANOSECONDS.toMillis(searchNanos), TimeUnit.NANOSECONDS.toMillis(openSearcherNanos));
        if (topDocs.totalHits == 0) {
            sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0);
            return sqr;
        }
        final DocsReader docsReader = new DocsReader();
        final EventAuthorizer authorizer = new EventAuthorizer() {

            @Override
            public boolean isAuthorized(ProvenanceEventRecord event) {
                return repository.isAuthorized(event, user);
            }

            @Override
            public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
                repository.authorize(event, user);
            }

            @Override
            public List<ProvenanceEventRecord> filterUnauthorizedEvents(List<ProvenanceEventRecord> events) {
                return repository.filterUnauthorizedEvents(events, user);
            }

            @Override
            public Set<ProvenanceEventRecord> replaceUnauthorizedWithPlaceholders(Set<ProvenanceEventRecord> events) {
                return repository.replaceUnauthorizedWithPlaceholders(events, user);
            }
        };
        matchingRecords = docsReader.read(topDocs, authorizer, searcher.getIndexSearcher().getIndexReader(), repository.getAllLogFiles(), retrievedCount, provenanceQuery.getMaxResults(), maxAttributeChars);
        final long readRecordsNanos = System.nanoTime() - finishSearch;
        logger.debug("Reading {} records took {} millis for {}", matchingRecords.size(), TimeUnit.NANOSECONDS.toMillis(readRecordsNanos), this);
        sqr.update(matchingRecords, topDocs.totalHits);
        final long queryNanos = System.nanoTime() - startNanos;
        logger.info("Successfully executed {} against Index {}; Search took {} milliseconds; Total Hits = {}", provenanceQuery, indexDirectory, TimeUnit.NANOSECONDS.toMillis(queryNanos), topDocs.totalHits);
        return sqr;
    } catch (final FileNotFoundException e) {
        // nothing has been indexed yet, or the data has already aged off
        logger.warn("Attempted to search Provenance Index {} but could not find the file due to {}", indexDirectory, e);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
        sqr.update(Collections.<ProvenanceEventRecord>emptyList(), 0);
        return sqr;
    } finally {
        if (searcher != null) {
            indexManager.returnIndexSearcher(searcher);
        }
    }
}

Example 2 with StandardQueryResult

use of org.apache.nifi.provenance.StandardQueryResult in project nifi by apache.

the class LuceneEventIndex method purgeObsoleteQueries.

private void purgeObsoleteQueries() {
    try {
        final Date now = new Date();
        final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator();
        while (queryIterator.hasNext()) {
            final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next();
            final StandardQueryResult result = entry.getValue().getResult();
            if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) {
                queryIterator.remove();
            }
        }
        final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator();
        while (lineageIterator.hasNext()) {
            final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next();
            final StandardLineageResult result = entry.getValue().getResult();
            if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) {
                lineageIterator.remove();
            }
        }
    } catch (final Exception e) {
        logger.error("Failed to expire Provenance Query Results due to {}", e.toString());
        logger.error("", e);
    }
}