Examples with SearchableField org.apache.nifi.provenance.search.SearchableField used on opensource projects

Example 1 with SearchableField

use of org.apache.nifi.provenance.search.SearchableField in project nifi by apache.

the class ControllerFacade method submitProvenance.

/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();
    // create the query
    final Query query = new Query(provenanceDto.getId());
    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;
                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }
        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }
        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }
        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());
        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }
    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());
    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}

Example 2 with SearchableField

use of org.apache.nifi.provenance.search.SearchableField in project nifi by apache.

the class ControllerFacade method getProvenanceSearchOptions.

/**
 * Gets the available options for searching provenance.
 *
 * @return the available options for searching provenance
 */
public ProvenanceOptionsDTO getProvenanceSearchOptions() {
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    // create the search options dto
    final ProvenanceOptionsDTO searchOptions = new ProvenanceOptionsDTO();
    final List<ProvenanceSearchableFieldDTO> searchableFieldNames = new ArrayList<>();
    final List<SearchableField> fields = provenanceRepository.getSearchableFields();
    for (final SearchableField field : fields) {
        // we prefer the user queries using startDate and endDate
        if (SearchableFields.EventTime.equals(field)) {
            continue;
        }
        final ProvenanceSearchableFieldDTO searchableField = new ProvenanceSearchableFieldDTO();
        searchableField.setId(field.getIdentifier());
        searchableField.setField(field.getSearchableFieldName());
        searchableField.setLabel(field.getFriendlyName());
        searchableField.setType(field.getFieldType().name());
        searchableFieldNames.add(searchableField);
    }
    final List<SearchableField> searchableAttributes = provenanceRepository.getSearchableAttributes();
    for (final SearchableField searchableAttr : searchableAttributes) {
        final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO();
        searchableAttribute.setId(searchableAttr.getIdentifier());
        searchableAttribute.setField(searchableAttr.getSearchableFieldName());
        searchableAttribute.setLabel(searchableAttr.getFriendlyName());
        searchableAttribute.setType(searchableAttr.getFieldType().name());
        searchableFieldNames.add(searchableAttribute);
    }
    searchOptions.setSearchableFields(searchableFieldNames);
    return searchOptions;
}

Example 3 with SearchableField

use of org.apache.nifi.provenance.search.SearchableField in project nifi by apache.

the class ConvertEventToLuceneDocument method convert.

public Document convert(final ProvenanceEventRecord record, final StorageSummary persistedEvent) {
    final Document doc = new Document();
    addField(doc, SearchableFields.FlowFileUUID, record.getFlowFileUuid());
    addField(doc, SearchableFields.Filename, record.getAttribute(CoreAttributes.FILENAME.key()));
    addField(doc, SearchableFields.ComponentID, record.getComponentId());
    addField(doc, SearchableFields.AlternateIdentifierURI, record.getAlternateIdentifierUri());
    addField(doc, SearchableFields.EventType, record.getEventType().name());
    addField(doc, SearchableFields.Relationship, record.getRelationship());
    addField(doc, SearchableFields.Details, record.getDetails());
    addField(doc, SearchableFields.ContentClaimSection, record.getContentClaimSection());
    addField(doc, SearchableFields.ContentClaimContainer, record.getContentClaimContainer());
    addField(doc, SearchableFields.ContentClaimIdentifier, record.getContentClaimIdentifier());
    addField(doc, SearchableFields.SourceQueueIdentifier, record.getSourceQueueIdentifier());
    addField(doc, SearchableFields.TransitURI, record.getTransitUri());
    for (final SearchableField searchableField : searchableAttributeFields) {
        addField(doc, searchableField, LuceneUtil.truncateIndexField(record.getAttribute(searchableField.getSearchableFieldName())));
    }
    // Index the fields that we always index (unless there's nothing else to index at all)
    if (!doc.getFields().isEmpty()) {
        // Always include Lineage Start Date because it allows us to make our Lineage queries more efficient.
        doc.add(new LongField(SearchableFields.LineageStartDate.getSearchableFieldName(), record.getLineageStartDate(), Store.NO));
        // Always include Event Time because most queries are bound by a start and end time.
        doc.add(new LongField(SearchableFields.EventTime.getSearchableFieldName(), record.getEventTime(), Store.NO));
        // We always include File Size because the UI wants to always render the controls for specifying this. This idea could be revisited.
        doc.add(new LongField(SearchableFields.FileSize.getSearchableFieldName(), record.getFileSize(), Store.NO));
        // We always store the event Event ID in the Document but do not index it. It doesn't make sense to query based on Event ID because
        // if we want a particular Event ID, we can just obtain it directly from the EventStore. But when we obtain a Document, this info must
        // be stored so that we know how to lookup the event in the store.
        doc.add(new UnIndexedLongField(SearchableFields.Identifier.getSearchableFieldName(), persistedEvent.getEventId()));
        // If it's event is a FORK, or JOIN, add the FlowFileUUID for all child/parent UUIDs.
        final ProvenanceEventType eventType = record.getEventType();
        if (eventType == ProvenanceEventType.FORK || eventType == ProvenanceEventType.CLONE || eventType == ProvenanceEventType.REPLAY) {
            for (final String uuid : record.getChildUuids()) {
                if (!uuid.equals(record.getFlowFileUuid())) {
                    addField(doc, SearchableFields.FlowFileUUID, uuid);
                }
            }
        } else if (eventType == ProvenanceEventType.JOIN) {
            for (final String uuid : record.getParentUuids()) {
                if (!uuid.equals(record.getFlowFileUuid())) {
                    addField(doc, SearchableFields.FlowFileUUID, uuid);
                }
            }
        } else if (eventType == ProvenanceEventType.RECEIVE && record.getSourceSystemFlowFileIdentifier() != null) {
            // If we get a receive with a Source System FlowFile Identifier, we add another Document that shows the UUID
            // that the Source System uses to refer to the data.
            final String sourceIdentifier = record.getSourceSystemFlowFileIdentifier();
            final String sourceFlowFileUUID;
            final int lastColon = sourceIdentifier.lastIndexOf(":");
            if (lastColon > -1 && lastColon < sourceIdentifier.length() - 2) {
                sourceFlowFileUUID = sourceIdentifier.substring(lastColon + 1);
            } else {
                sourceFlowFileUUID = null;
            }
            if (sourceFlowFileUUID != null) {
                addField(doc, SearchableFields.FlowFileUUID, sourceFlowFileUUID);
            }
        }
        return doc;
    }
    return null;
}

Example 4 with SearchableField

use of org.apache.nifi.provenance.search.SearchableField in project nifi by apache.

the class IndexingAction method index.

public void index(final StandardProvenanceEventRecord record, final IndexWriter indexWriter, final Integer blockIndex) throws IOException {
    final Document doc = new Document();
    addField(doc, SearchableFields.FlowFileUUID, record.getFlowFileUuid(), Store.NO);
    addField(doc, SearchableFields.Filename, record.getAttribute(CoreAttributes.FILENAME.key()), Store.NO);
    addField(doc, SearchableFields.ComponentID, record.getComponentId(), Store.NO);
    addField(doc, SearchableFields.AlternateIdentifierURI, record.getAlternateIdentifierUri(), Store.NO);
    addField(doc, SearchableFields.EventType, record.getEventType().name(), Store.NO);
    addField(doc, SearchableFields.Relationship, record.getRelationship(), Store.NO);
    addField(doc, SearchableFields.Details, record.getDetails(), Store.NO);
    addField(doc, SearchableFields.ContentClaimSection, record.getContentClaimSection(), Store.NO);
    addField(doc, SearchableFields.ContentClaimContainer, record.getContentClaimContainer(), Store.NO);
    addField(doc, SearchableFields.ContentClaimIdentifier, record.getContentClaimIdentifier(), Store.NO);
    addField(doc, SearchableFields.SourceQueueIdentifier, record.getSourceQueueIdentifier(), Store.NO);
    addField(doc, SearchableFields.TransitURI, record.getTransitUri(), Store.NO);
    for (final SearchableField searchableField : searchableAttributeFields) {
        addField(doc, searchableField, LuceneUtil.truncateIndexField(record.getAttribute(searchableField.getSearchableFieldName())), Store.NO);
    }
    final String storageFilename = LuceneUtil.substringBefore(record.getStorageFilename(), ".");
    // Index the fields that we always index (unless there's nothing else to index at all)
    if (!doc.getFields().isEmpty()) {
        doc.add(new LongField(SearchableFields.LineageStartDate.getSearchableFieldName(), record.getLineageStartDate(), Store.NO));
        doc.add(new LongField(SearchableFields.EventTime.getSearchableFieldName(), record.getEventTime(), Store.NO));
        doc.add(new LongField(SearchableFields.FileSize.getSearchableFieldName(), record.getFileSize(), Store.NO));
        doc.add(new StringField(FieldNames.STORAGE_FILENAME, storageFilename, Store.YES));
        if (blockIndex == null) {
            doc.add(new LongField(FieldNames.STORAGE_FILE_OFFSET, record.getStorageByteOffset(), Store.YES));
        } else {
            doc.add(new IntField(FieldNames.BLOCK_INDEX, blockIndex, Store.YES));
            doc.add(new LongField(SearchableFields.Identifier.getSearchableFieldName(), record.getEventId(), Store.YES));
        }
        // If it's event is a FORK, or JOIN, add the FlowFileUUID for all child/parent UUIDs.
        final ProvenanceEventType eventType = record.getEventType();
        if (eventType == ProvenanceEventType.FORK || eventType == ProvenanceEventType.CLONE || eventType == ProvenanceEventType.REPLAY) {
            for (final String uuid : record.getChildUuids()) {
                if (!uuid.equals(record.getFlowFileUuid())) {
                    addField(doc, SearchableFields.FlowFileUUID, uuid, Store.NO);
                }
            }
        } else if (eventType == ProvenanceEventType.JOIN) {
            for (final String uuid : record.getParentUuids()) {
                if (!uuid.equals(record.getFlowFileUuid())) {
                    addField(doc, SearchableFields.FlowFileUUID, uuid, Store.NO);
                }
            }
        } else if (eventType == ProvenanceEventType.RECEIVE && record.getSourceSystemFlowFileIdentifier() != null) {
            // If we get a receive with a Source System FlowFile Identifier, we add another Document that shows the UUID
            // that the Source System uses to refer to the data.
            final String sourceIdentifier = record.getSourceSystemFlowFileIdentifier();
            final String sourceFlowFileUUID;
            final int lastColon = sourceIdentifier.lastIndexOf(":");
            if (lastColon > -1 && lastColon < sourceIdentifier.length() - 2) {
                sourceFlowFileUUID = sourceIdentifier.substring(lastColon + 1);
            } else {
                sourceFlowFileUUID = null;
            }
            if (sourceFlowFileUUID != null) {
                addField(doc, SearchableFields.FlowFileUUID, sourceFlowFileUUID, Store.NO);
            }
        }
        indexWriter.addDocument(doc);
    }
}

Example 5 with SearchableField

use of org.apache.nifi.provenance.search.SearchableField in project nifi by apache.

the class VolatileProvenanceRepository method createFilter.

private Filter<ProvenanceEventRecord> createFilter(final Query query, final NiFiUser user) {
    return new Filter<ProvenanceEventRecord>() {

        @Override
        public boolean select(final ProvenanceEventRecord event) {
            if (!isAuthorized(event, user)) {
                return false;
            }
            if (query.getStartDate() != null && query.getStartDate().getTime() > event.getEventTime()) {
                return false;
            }
            if (query.getEndDate() != null && query.getEndDate().getTime() < event.getEventTime()) {
                return false;
            }
            if (query.getMaxFileSize() != null) {
                final long maxFileSize = DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue();
                if (event.getFileSize() > maxFileSize) {
                    return false;
                }
            }
            if (query.getMinFileSize() != null) {
                final long minFileSize = DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue();
                if (event.getFileSize() < minFileSize) {
                    return false;
                }
            }
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                final SearchableField searchableField = searchTerm.getSearchableField();
                final String searchValue = searchTerm.getValue();
                if (searchableField.isAttribute()) {
                    final String attributeName = searchableField.getIdentifier();
                    final String eventAttributeValue = event.getAttributes().get(attributeName);
                    if (searchValue.contains("?") || searchValue.contains("*")) {
                        if (eventAttributeValue == null || eventAttributeValue.isEmpty()) {
                            return false;
                        }
                        final String regex = searchValue.replace("?", ".").replace("*", ".*");
                        final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                        if (!pattern.matcher(eventAttributeValue).matches()) {
                            return false;
                        }
                    } else if (!searchValue.equalsIgnoreCase(eventAttributeValue)) {
                        return false;
                    }
                } else {
                    // if FlowFileUUID, search parent & child UUID's also.
                    if (searchableField.equals(SearchableFields.FlowFileUUID)) {
                        if (searchValue.contains("?") || searchValue.contains("*")) {
                            final String regex = searchValue.replace("?", ".").replace("*", ".*");
                            final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                            if (pattern.matcher(event.getFlowFileUuid()).matches()) {
                                continue;
                            }
                            boolean found = false;
                            for (final String uuid : event.getParentUuids()) {
                                if (pattern.matcher(uuid).matches()) {
                                    found = true;
                                    break;
                                }
                            }
                            for (final String uuid : event.getChildUuids()) {
                                if (pattern.matcher(uuid).matches()) {
                                    found = true;
                                    break;
                                }
                            }
                            if (found) {
                                continue;
                            }
                        } else if (event.getFlowFileUuid().equals(searchValue) || event.getParentUuids().contains(searchValue) || event.getChildUuids().contains(searchValue)) {
                            continue;
                        }
                        return false;
                    }
                    final Object fieldValue = getFieldValue(event, searchableField);
                    if (fieldValue == null) {
                        return false;
                    }
                    if (searchValue.contains("?") || searchValue.contains("*")) {
                        final String regex = searchValue.replace("?", ".").replace("*", ".*");
                        final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
                        if (!pattern.matcher(String.valueOf(fieldValue)).matches()) {
                            return false;
                        }
                    } else if (!searchValue.equalsIgnoreCase(String.valueOf(fieldValue))) {
                        return false;
                    }
                }
            }
            return true;
        }
    };
}