Examples with ProvenanceRequestDTO org.apache.nifi.web.api.dto.provenance.ProvenanceRequestDTO used on opensource projects

Example 1 with ProvenanceRequestDTO

use of org.apache.nifi.web.api.dto.provenance.ProvenanceRequestDTO in project nifi by apache.

the class ControllerFacade method getProvenanceQuery.

/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());
        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }
        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();
        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();
        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);
        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }
        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());
        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }
        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }
        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }
        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));
        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }
        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}

Example 2 with ProvenanceRequestDTO

use of org.apache.nifi.web.api.dto.provenance.ProvenanceRequestDTO in project nifi by apache.

the class ControllerFacade method submitProvenance.

/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();
    // create the query
    final Query query = new Query(provenanceDto.getId());
    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;
                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }
        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }
        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }
        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());
        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }
    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());
    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}

Example 3 with ProvenanceRequestDTO

use of org.apache.nifi.web.api.dto.provenance.ProvenanceRequestDTO in project nifi by apache.

the class ProvenanceQueryEndpointMerger method mergeResponses.

protected void mergeResponses(ProvenanceDTO clientDto, Map<NodeIdentifier, ProvenanceDTO> dtoMap, Set<NodeResponse> successfulResponses, Set<NodeResponse> problematicResponses) {
    final ProvenanceResultsDTO results = clientDto.getResults();
    final ProvenanceRequestDTO request = clientDto.getRequest();
    final List<ProvenanceEventDTO> allResults = new ArrayList<>(1024);
    final Set<String> errors = new HashSet<>();
    Date oldestEventDate = new Date();
    int percentageComplete = 0;
    boolean finished = true;
    long totalRecords = 0;
    for (final Map.Entry<NodeIdentifier, ProvenanceDTO> entry : dtoMap.entrySet()) {
        final NodeIdentifier nodeIdentifier = entry.getKey();
        final String nodeAddress = nodeIdentifier.getApiAddress() + ":" + nodeIdentifier.getApiPort();
        final ProvenanceDTO nodeDto = entry.getValue();
        final ProvenanceResultsDTO nodeResultDto = nodeDto.getResults();
        if (nodeResultDto != null && nodeResultDto.getProvenanceEvents() != null) {
            // increment the total number of records
            totalRecords += nodeResultDto.getTotalCount();
            // populate the cluster identifier
            for (final ProvenanceEventDTO eventDto : nodeResultDto.getProvenanceEvents()) {
                // from the Cluster Coordinator.
                if (eventDto.getClusterNodeId() == null || eventDto.getClusterNodeAddress() == null) {
                    eventDto.setClusterNodeId(nodeIdentifier.getId());
                    eventDto.setClusterNodeAddress(nodeAddress);
                    // add node identifier to the event's id so that it is unique across cluster
                    eventDto.setId(nodeIdentifier.getId() + eventDto.getId());
                }
                allResults.add(eventDto);
            }
        }
        if (nodeResultDto.getOldestEvent() != null && nodeResultDto.getOldestEvent().before(oldestEventDate)) {
            oldestEventDate = nodeResultDto.getOldestEvent();
        }
        if (nodeResultDto.getErrors() != null) {
            for (final String error : nodeResultDto.getErrors()) {
                errors.add(nodeAddress + " -- " + error);
            }
        }
        percentageComplete += nodeDto.getPercentCompleted();
        if (!nodeDto.isFinished()) {
            finished = false;
        }
    }
    percentageComplete /= dtoMap.size();
    // consider any problematic responses as errors
    for (final NodeResponse problematicResponse : problematicResponses) {
        final NodeIdentifier problemNode = problematicResponse.getNodeId();
        final String problemNodeAddress = problemNode.getApiAddress() + ":" + problemNode.getApiPort();
        errors.add(String.format("%s -- Request did not complete successfully (Status code: %s)", problemNodeAddress, problematicResponse.getStatus()));
    }
    // Since we get back up to the maximum number of results from each node, we need to sort those values and then
    // grab only the first X number of them. We do a sort based on time, such that the newest are included.
    // If 2 events have the same timestamp, we do a secondary sort based on Cluster Node Identifier. If those are
    // equal, we perform a tertiary sort based on the the event id
    Collections.sort(allResults, new Comparator<ProvenanceEventDTO>() {

        @Override
        public int compare(final ProvenanceEventDTO o1, final ProvenanceEventDTO o2) {
            final int eventTimeComparison = o1.getEventTime().compareTo(o2.getEventTime());
            if (eventTimeComparison != 0) {
                return -eventTimeComparison;
            }
            final String nodeId1 = o1.getClusterNodeId();
            final String nodeId2 = o2.getClusterNodeId();
            final int nodeIdComparison;
            if (nodeId1 == null && nodeId2 == null) {
                nodeIdComparison = 0;
            } else if (nodeId1 == null) {
                nodeIdComparison = 1;
            } else if (nodeId2 == null) {
                nodeIdComparison = -1;
            } else {
                nodeIdComparison = -nodeId1.compareTo(nodeId2);
            }
            if (nodeIdComparison != 0) {
                return nodeIdComparison;
            }
            return -Long.compare(o1.getEventId(), o2.getEventId());
        }
    });
    final int maxResults = request.getMaxResults().intValue();
    final List<ProvenanceEventDTO> selectedResults;
    if (allResults.size() < maxResults) {
        selectedResults = allResults;
    } else {
        selectedResults = allResults.subList(0, maxResults);
    }
    // include any errors
    if (errors.size() > 0) {
        results.setErrors(errors);
    }
    if (clientDto.getRequest().getMaxResults() != null && totalRecords >= clientDto.getRequest().getMaxResults()) {
        results.setTotalCount(clientDto.getRequest().getMaxResults().longValue());
        results.setTotal(FormatUtils.formatCount(clientDto.getRequest().getMaxResults().longValue()) + "+");
    } else {
        results.setTotal(FormatUtils.formatCount(totalRecords));
        results.setTotalCount(totalRecords);
    }
    results.setProvenanceEvents(selectedResults);
    results.setOldestEvent(oldestEventDate);
    results.setGenerated(new Date());
    clientDto.setPercentCompleted(percentageComplete);
    clientDto.setFinished(finished);
}