Examples with SearchTerm org.apache.nifi.provenance.search.SearchTerm used on opensource projects

Example 1 with SearchTerm

use of org.apache.nifi.provenance.search.SearchTerm in project nifi by apache.

the class ControllerFacade method getProvenanceQuery.

/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());
        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }
        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();
        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();
        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);
        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }
        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());
        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }
        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }
        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }
        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));
        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }
        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}

Example 2 with SearchTerm

use of org.apache.nifi.provenance.search.SearchTerm in project nifi by apache.

the class LatestEventsPerProcessorQuery method evaluate.

@Override
public Optional<List<Long>> evaluate(final Query query) {
    if (query.getMaxResults() > 1000) {
        // If query max results > 1000 then we know we don't have enough results. So just return empty.
        return Optional.empty();
    }
    final List<SearchTerm> terms = query.getSearchTerms();
    if (terms.size() != 1) {
        return Optional.empty();
    }
    final SearchTerm term = terms.get(0);
    if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) {
        return Optional.empty();
    }
    if (query.getEndDate() != null || query.getStartDate() != null) {
        return Optional.empty();
    }
    final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue());
    if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) {
        return Optional.empty();
    }
    List<Long> eventIds = ringBuffer.asList();
    if (eventIds.size() > query.getMaxResults()) {
        eventIds = eventIds.subList(0, query.getMaxResults());
    }
    return Optional.of(eventIds);
}

Example 3 with SearchTerm

use of org.apache.nifi.provenance.search.SearchTerm in project nifi by apache.

the class LuceneUtil method convertQuery.

public static org.apache.lucene.search.Query convertQuery(final org.apache.nifi.provenance.search.Query query) {
    if (query.getStartDate() == null && query.getEndDate() == null && query.getSearchTerms().isEmpty()) {
        return new MatchAllDocsQuery();
    }
    final BooleanQuery luceneQuery = new BooleanQuery();
    for (final SearchTerm searchTerm : query.getSearchTerms()) {
        final String searchValue = searchTerm.getValue();
        if (searchValue == null) {
            throw new IllegalArgumentException("Empty search value not allowed (for term '" + searchTerm.getSearchableField().getFriendlyName() + "')");
        }
        if (searchValue.contains("*") || searchValue.contains("?")) {
            luceneQuery.add(new BooleanClause(new WildcardQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST));
        } else {
            luceneQuery.add(new BooleanClause(new TermQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST));
        }
    }
    final Long minBytes = query.getMinFileSize() == null ? null : DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue();
    final Long maxBytes = query.getMaxFileSize() == null ? null : DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue();
    if (minBytes != null || maxBytes != null) {
        luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.FileSize.getSearchableFieldName(), minBytes, maxBytes, true, true), Occur.MUST);
    }
    final Long minDateTime = query.getStartDate() == null ? null : query.getStartDate().getTime();
    final Long maxDateTime = query.getEndDate() == null ? null : query.getEndDate().getTime();
    if (maxDateTime != null || minDateTime != null) {
        luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.EventTime.getSearchableFieldName(), minDateTime, maxDateTime, true, true), Occur.MUST);
    }
    return luceneQuery;
}

Example 4 with SearchTerm

use of org.apache.nifi.provenance.search.SearchTerm in project nifi by apache.

the class VolatileProvenanceRepository method createFilter.

private Filter<ProvenanceEventRecord> createFilter(final Query query, final NiFiUser user) {
    return new Filter<ProvenanceEventRecord>() {

        @Override
        public boolean select(final ProvenanceEventRecord event) {
            if (!isAuthorized(event, user)) {
                return false;
            }
            if (query.getStartDate() != null && query.getStartDate().getTime() > event.getEventTime()) {
                return false;
            }
            if (query.getEndDate() != null && query.getEndDate().getTime() < event.getEventTime()) {
                return false;
            }
            if (query.getMaxFileSize() != null) {
                final long maxFileSize = DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue();
                if (event.getFileSize() > maxFileSize) {
                    return false;
                }
            }
            if (query.getMinFileSize() != null) {
                final long minFileSize = DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue();
                if (event.getFileSize() < minFileSize) {
                    return false;
                }
            }
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                final SearchableField searchableField = searchTerm.getSearchableField();
                final String searchValue = searchTerm.getValue();
                if (searchableField.isAttribute()) {
                    final String attributeName = searchableField.getIdentifier();
                    final String eventAttributeValue = event.getAttributes().get(attributeName);
                    if (searchValue.contains("?") || searchValue.contains("*")) {
                        if (eventAttributeValue == null || eventAttributeValue.isEmpty()) {
                            return false;
                        }
                        final String regex = searchValue.replace("?", ".").replace("*", ".*");
                        final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                        if (!pattern.matcher(eventAttributeValue).matches()) {
                            return false;
                        }
                    } else if (!searchValue.equalsIgnoreCase(eventAttributeValue)) {
                        return false;
                    }
                } else {
                    // if FlowFileUUID, search parent & child UUID's also.
                    if (searchableField.equals(SearchableFields.FlowFileUUID)) {
                        if (searchValue.contains("?") || searchValue.contains("*")) {
                            final String regex = searchValue.replace("?", ".").replace("*", ".*");
                            final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                            if (pattern.matcher(event.getFlowFileUuid()).matches()) {
                                continue;
                            }
                            boolean found = false;
                            for (final String uuid : event.getParentUuids()) {
                                if (pattern.matcher(uuid).matches()) {
                                    found = true;
                                    break;
                                }
                            }
                            for (final String uuid : event.getChildUuids()) {
                                if (pattern.matcher(uuid).matches()) {
                                    found = true;
                                    break;
                                }
                            }
                            if (found) {
                                continue;
                            }
                        } else if (event.getFlowFileUuid().equals(searchValue) || event.getParentUuids().contains(searchValue) || event.getChildUuids().contains(searchValue)) {
                            continue;
                        }
                        return false;
                    }
                    final Object fieldValue = getFieldValue(event, searchableField);
                    if (fieldValue == null) {
                        return false;
                    }
                    if (searchValue.contains("?") || searchValue.contains("*")) {
                        final String regex = searchValue.replace("?", ".").replace("*", ".*");
                        final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                        if (!pattern.matcher(String.valueOf(fieldValue)).matches()) {
                            return false;
                        }
                    } else if (!searchValue.equalsIgnoreCase(String.valueOf(fieldValue))) {
                        return false;
                    }
                }
            }
            return true;
        }
    };
}