Examples with ProvenanceRepository org.apache.nifi.provenance.ProvenanceRepository used on opensource projects

Example 1 with ProvenanceRepository

use of org.apache.nifi.provenance.ProvenanceRepository in project nifi by apache.

the class ControllerFacade method getProvenanceQuery.

/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());
        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }
        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();
        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();
        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);
        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }
        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());
        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }
        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }
        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }
        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));
        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }
        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}

Example 2 with ProvenanceRepository

use of org.apache.nifi.provenance.ProvenanceRepository in project nifi by apache.

the class ControllerFacade method submitProvenance.

/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();
    // create the query
    final Query query = new Query(provenanceDto.getId());
    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;
                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }
        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }
        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }
        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());
        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }
    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());
    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}

Example 3 with ProvenanceRepository

use of org.apache.nifi.provenance.ProvenanceRepository in project nifi by apache.

the class ControllerFacade method getProvenanceSearchOptions.

/**
 * Gets the available options for searching provenance.
 *
 * @return the available options for searching provenance
 */
public ProvenanceOptionsDTO getProvenanceSearchOptions() {
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    // create the search options dto
    final ProvenanceOptionsDTO searchOptions = new ProvenanceOptionsDTO();
    final List<ProvenanceSearchableFieldDTO> searchableFieldNames = new ArrayList<>();
    final List<SearchableField> fields = provenanceRepository.getSearchableFields();
    for (final SearchableField field : fields) {
        // we prefer the user queries using startDate and endDate
        if (SearchableFields.EventTime.equals(field)) {
            continue;
        }
        final ProvenanceSearchableFieldDTO searchableField = new ProvenanceSearchableFieldDTO();
        searchableField.setId(field.getIdentifier());
        searchableField.setField(field.getSearchableFieldName());
        searchableField.setLabel(field.getFriendlyName());
        searchableField.setType(field.getFieldType().name());
        searchableFieldNames.add(searchableField);
    }
    final List<SearchableField> searchableAttributes = provenanceRepository.getSearchableAttributes();
    for (final SearchableField searchableAttr : searchableAttributes) {
        final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO();
        searchableAttribute.setId(searchableAttr.getIdentifier());
        searchableAttribute.setField(searchableAttr.getSearchableFieldName());
        searchableAttribute.setLabel(searchableAttr.getFriendlyName());
        searchableAttribute.setType(searchableAttr.getFieldType().name());
        searchableFieldNames.add(searchableAttribute);
    }
    searchOptions.setSearchableFields(searchableFieldNames);
    return searchOptions;
}

Example 4 with ProvenanceRepository

use of org.apache.nifi.provenance.ProvenanceRepository in project nifi by apache.

the class ControllerFacade method getLineage.

/**
 * Gets the lineage with the specified id.
 *
 * @param lineageId id
 * @return the lineage with the specified id
 */
public LineageDTO getLineage(final String lineageId) {
    // get the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final ComputeLineageSubmission computeLineageSubmission = provenanceRepository.retrieveLineageSubmission(lineageId, NiFiUserUtils.getNiFiUser());
    // ensure the submission was found
    if (computeLineageSubmission == null) {
        throw new ResourceNotFoundException("Cannot find the results for the specified lineage request. Results may have been purged.");
    }
    return dtoFactory.createLineageDto(computeLineageSubmission);
}

Example 5 with ProvenanceRepository

use of org.apache.nifi.provenance.ProvenanceRepository in project nifi by apache.

the class ControllerFacade method deleteLineage.

/**
 * Deletes the lineage with the specified id.
 *
 * @param lineageId id
 */
public void deleteLineage(final String lineageId) {
    // get the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final ComputeLineageSubmission computeLineageSubmission = provenanceRepository.retrieveLineageSubmission(lineageId, NiFiUserUtils.getNiFiUser());
    if (computeLineageSubmission != null) {
        computeLineageSubmission.cancel();
    }
}