Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 1 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project kylo by Teradata.

the class FeedEventStatistics method finishedEvent.

public void finishedEvent(ProvenanceEventRecord event, Long eventId) {
    String feedFlowFileId = allFlowFileToFeedFlowFile.get(event.getFlowFileUuid());
    if (feedFlowFileId != null && ProvenanceEventType.DROP.equals(event.getEventType())) {
        // get the feed flow fileId for this event
        AtomicInteger activeCounts = feedFlowProcessing.get(feedFlowFileId);
        if (activeCounts != null) {
            feedFlowProcessing.get(feedFlowFileId).decrementAndGet();
            if (activeCounts.get() <= 0) {
                // Feed is finished
                eventsThatCompleteFeedFlow.add(eventId);
                feedFlowFileEndTime.put(feedFlowFileId, event.getEventTime());
                decrementRunningProcessorFeedFlows(feedFlowFileId);
            }
        }
    }
    if (feedFlowFileId != null && ProvenanceEventUtil.isTerminatedByFailureRelationship(event)) {
        // add to failureMap
        feedFlowFileFailureCount.computeIfAbsent(feedFlowFileId, flowFileId -> new AtomicInteger(0)).incrementAndGet();
    }
}

Example 2 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project kylo by Teradata.

the class SimulateNiFiFlow method createSplit.

public void createSplit() {
    String flowFileId = UUID.randomUUID().toString();
    Long entryDate = DateTime.now().getMillis();
    ProvenanceEventRecord start = buildEvent(componentIdSplit1, ProvenanceEventType.CREATE, flowFileId, entryDate);
    ProvenanceEventRecord next = buildEvent(componentIdSplit2, ProvenanceEventType.ATTRIBUTES_MODIFIED, flowFileId, entryDate);
    List<ProvenanceEventRecord> split = buildSplitEvent(componentIdSplitParent, flowFileId, entryDate, 3);
    ProvenanceEventRecord last = buildEvent(componentIdSplitFlowEnd, ProvenanceEventType.DROP, flowFileId, entryDate);
    addToFlow(start);
    addToFlow(next, 1000L);
    ProvenanceEventRecord parentSplit = split.stream().filter(e -> e.getEventType().equals(ProvenanceEventType.CLONE)).findFirst().orElse(null);
    split.stream().forEach(e -> addToFlow(e));
    addToFlow(last, 1000L);
    // drop the children
    for (String childId : parentSplit.getChildUuids()) {
        ProvenanceEventRecord childEnd = buildEvent(componentIdSplitChild2, ProvenanceEventType.DROP, childId, entryDate);
        addToFlow(childEnd, 1000L);
    }
}

Example 3 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project kylo by Teradata.

the class SimulateNiFiFlow method buildEvent.

private ProvenanceEventRecord buildEvent(String componentId, ProvenanceEventType type, String flowfileId, Long entryDate) {
    StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
    ProvenanceEventRecord eventRecord = builder.setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(entryDate).setComponentId(componentId).setComponentType(componentType).setCurrentContentClaim("container", "section", "identifier", 0L, 0L).setFlowFileUUID(flowfileId).setEventType(type).build();
    return eventRecord;
}

Example 4 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project kylo by Teradata.

the class SimulateNiFiFlow method buildSplitEvent.

private List<ProvenanceEventRecord> buildSplitEvent(String componentId, String flowfileId, Long entryDate, Integer children) {
    List<ProvenanceEventRecord> allFlowFiles = new ArrayList<>();
    List<ProvenanceEventRecord> childFlowFiles = new ArrayList<>();
    List<String> parentFlowFileIds = new ArrayList<>();
    parentFlowFileIds.add(flowfileId);
    List<String> childFlowFileIds = new ArrayList<>();
    for (int i = 0; i < children; i++) {
        String ffId = UUID.randomUUID().toString();
        ProvenanceEventRecord updateEvent = buildEvent(componentIdSplitChild1, ProvenanceEventType.ATTRIBUTES_MODIFIED, ffId, DateTime.now().getMillis());
        childFlowFileIds.add(ffId);
        childFlowFiles.add(updateEvent);
    }
    StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(entryDate).setEventType(ProvenanceEventType.CLONE).setComponentId(componentId).setComponentType(componentType).setCurrentContentClaim("container", "section", "identifier", 0L, 0L).setFlowFileUUID(flowfileId);
    childFlowFileIds.stream().forEach(id -> builder.addChildUuid(id));
    parentFlowFileIds.stream().forEach(id -> builder.addParentUuid(id));
    // .setChildUuids(childFlowFileIds)
    // .setParentUuids(parentFlowFileIds)
    ProvenanceEventRecord eventRecord = builder.build();
    allFlowFiles.add(eventRecord);
    allFlowFiles.addAll(childFlowFiles);
    return allFlowFiles;
}

Example 5 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class ControllerFacade method getProvenanceQuery.

/**
 * Retrieves the results of a provenance query.
 *
 * @param provenanceId id
 * @return the results of a provenance query
 */
public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) {
    try {
        // get the query to the provenance repository
        final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
        final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser());
        // ensure the query results could be found
        if (querySubmission == null) {
            throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged.");
        }
        // get the original query and the results
        final Query query = querySubmission.getQuery();
        final QueryResult queryResult = querySubmission.getResult();
        // build the response
        final ProvenanceDTO provenanceDto = new ProvenanceDTO();
        final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
        final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO();
        // include the original request and results
        provenanceDto.setRequest(requestDto);
        provenanceDto.setResults(resultsDto);
        // convert the original request
        requestDto.setStartDate(query.getStartDate());
        requestDto.setEndDate(query.getEndDate());
        requestDto.setMinimumFileSize(query.getMinFileSize());
        requestDto.setMaximumFileSize(query.getMaxFileSize());
        requestDto.setMaxResults(query.getMaxResults());
        if (query.getSearchTerms() != null) {
            final Map<String, String> searchTerms = new HashMap<>();
            for (final SearchTerm searchTerm : query.getSearchTerms()) {
                searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue());
            }
            requestDto.setSearchTerms(searchTerms);
        }
        // convert the provenance
        provenanceDto.setId(query.getIdentifier());
        provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime());
        provenanceDto.setExpiration(queryResult.getExpiration());
        provenanceDto.setFinished(queryResult.isFinished());
        provenanceDto.setPercentCompleted(queryResult.getPercentComplete());
        // convert each event
        final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults);
        if (includeResults || queryResult.isFinished()) {
            final List<ProvenanceEventDTO> events = new ArrayList<>();
            for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) {
                events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize)));
            }
            resultsDto.setProvenanceEvents(events);
        }
        if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
            resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
            resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
        } else {
            resultsDto.setTotalCount(queryResult.getTotalHitCount());
            resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
        }
        // include any errors
        if (queryResult.getError() != null) {
            final Set<String> errors = new HashSet<>();
            errors.add(queryResult.getError());
            resultsDto.setErrors(errors);
        }
        // set the generated timestamp
        final Date now = new Date();
        resultsDto.setGenerated(now);
        resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime()));
        // get the oldest available event time
        final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1);
        if (!firstEvent.isEmpty()) {
            resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime()));
        }
        provenanceDto.setResults(resultsDto);
        return provenanceDto;
    } catch (final IOException ioe) {
        throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe);
    }
}