Examples with AsyncQuerySubmission org.apache.nifi.provenance.AsyncQuerySubmission used on opensource projects

Example 1 with AsyncQuerySubmission

use of org.apache.nifi.provenance.AsyncQuerySubmission in project nifi by apache.

the class LuceneEventIndex method submitQuery.

@Override
public QuerySubmission submitQuery(final Query query, final EventAuthorizer authorizer, final String userId) {
    validate(query);
    // Check if we have any cached queries first that can give us the answer
    for (final CachedQuery cachedQuery : cachedQueries) {
        final Optional<List<Long>> eventIdListOption = cachedQuery.evaluate(query);
        if (eventIdListOption.isPresent()) {
            final AsyncQuerySubmission submission = new AsyncQuerySubmission(query, 1, userId);
            querySubmissionMap.put(query.getIdentifier(), submission);
            final List<Long> eventIds = eventIdListOption.get();
            queryExecutor.submit(() -> {
                List<ProvenanceEventRecord> events;
                try {
                    events = eventStore.getEvents(eventIds, authorizer, EventTransformer.EMPTY_TRANSFORMER);
                    submission.getResult().update(events, eventIds.size());
                } catch (final Exception e) {
                    submission.getResult().setError("Failed to retrieve Provenance Events from store; see logs for more details");
                    logger.error("Failed to retrieve Provenance Events from store", e);
                }
            });
            // up to 500 milliseconds to complete before running.
            try {
                submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException e) {
                Thread.currentThread().interrupt();
            }
            return submission;
        }
    }
    final List<File> indexDirectories = directoryManager.getDirectories(query.getStartDate() == null ? null : query.getStartDate().getTime(), query.getEndDate() == null ? null : query.getEndDate().getTime());
    final AsyncQuerySubmission submission = new AsyncQuerySubmission(query, indexDirectories.size(), userId);
    querySubmissionMap.put(query.getIdentifier(), submission);
    final org.apache.lucene.search.Query luceneQuery = LuceneUtil.convertQuery(query);
    logger.debug("Submitting query {} with identifier {} against index directories {}", luceneQuery, query.getIdentifier(), indexDirectories);
    if (indexDirectories.isEmpty()) {
        submission.getResult().update(Collections.emptyList(), 0L);
    } else {
        Collections.sort(indexDirectories, DirectoryUtils.NEWEST_INDEX_FIRST);
        for (final File indexDir : indexDirectories) {
            queryExecutor.submit(new QueryTask(luceneQuery, submission.getResult(), query.getMaxResults(), indexManager, indexDir, eventStore, authorizer, EventTransformer.EMPTY_TRANSFORMER));
        }
    }
    // up to 500 milliseconds to complete before running.
    try {
        submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS);
    } catch (final InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    return submission;
}

Example 2 with AsyncQuerySubmission

use of org.apache.nifi.provenance.AsyncQuerySubmission in project nifi by apache.

the class LuceneEventIndex method purgeObsoleteQueries.

private void purgeObsoleteQueries() {
    try {
        final Date now = new Date();
        final Iterator<Map.Entry<String, AsyncQuerySubmission>> queryIterator = querySubmissionMap.entrySet().iterator();
        while (queryIterator.hasNext()) {
            final Map.Entry<String, AsyncQuerySubmission> entry = queryIterator.next();
            final StandardQueryResult result = entry.getValue().getResult();
            if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) {
                queryIterator.remove();
            }
        }
        final Iterator<Map.Entry<String, AsyncLineageSubmission>> lineageIterator = lineageSubmissionMap.entrySet().iterator();
        while (lineageIterator.hasNext()) {
            final Map.Entry<String, AsyncLineageSubmission> entry = lineageIterator.next();
            final StandardLineageResult result = entry.getValue().getResult();
            if (entry.getValue().isCanceled() || result.isFinished() && result.getExpiration().before(now)) {
                lineageIterator.remove();
            }
        }
    } catch (final Exception e) {
        logger.error("Failed to expire Provenance Query Results due to {}", e.toString());
        logger.error("", e);
    }
}