Examples with AsyncLineageSubmission org.apache.nifi.provenance.AsyncLineageSubmission used on opensource projects

Example 1 with AsyncLineageSubmission

use of org.apache.nifi.provenance.AsyncLineageSubmission in project nifi by apache.

the class LuceneEventIndex method retrieveLineageSubmission.

@Override
public AsyncLineageSubmission retrieveLineageSubmission(final String lineageIdentifier, final NiFiUser user) {
    final AsyncLineageSubmission submission = lineageSubmissionMap.get(lineageIdentifier);
    final String userId = submission.getSubmitterIdentity();
    if (user == null && userId == null) {
        return submission;
    }
    if (user == null) {
        throw new AccessDeniedException("Cannot retrieve Provenance Lineage Submission because no user id was provided");
    }
    if (userId == null || userId.equals(user.getIdentity())) {
        return submission;
    }
    throw new AccessDeniedException("Cannot retrieve Provenance Lineage Submission because " + user.getIdentity() + " is not the user who submitted the request");
}

Example 2 with AsyncLineageSubmission

use of org.apache.nifi.provenance.AsyncLineageSubmission in project nifi by apache.

the class LuceneEventIndex method submitLineageComputation.

@Override
public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user, final EventAuthorizer eventAuthorizer) {
    final Optional<ProvenanceEventRecord> eventOption;
    try {
        eventOption = eventStore.getEvent(eventId);
    } catch (final Exception e) {
        logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e);
        final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity());
        result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information.");
        return result;
    }
    if (!eventOption.isPresent()) {
        final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity());
        result.getResult().setError("Could not find Provenance Event with ID " + eventId);
        lineageSubmissionMap.put(result.getLineageIdentifier(), result);
        return result;
    }
    final ProvenanceEventRecord event = eventOption.get();
    return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, eventAuthorizer, LineageComputationType.FLOWFILE_LINEAGE, eventId, event.getLineageStartDate(), Long.MAX_VALUE);
}

Example 3 with AsyncLineageSubmission

use of org.apache.nifi.provenance.AsyncLineageSubmission in project nifi by apache.

the class LuceneEventIndex method submitLineageComputation.

private ComputeLineageSubmission submitLineageComputation(final Collection<String> flowFileUuids, final NiFiUser user, final EventAuthorizer eventAuthorizer, final LineageComputationType computationType, final Long eventId, final long startTimestamp, final long endTimestamp) {
    final List<File> indexDirs = directoryManager.getDirectories(startTimestamp, endTimestamp);
    final AsyncLineageSubmission submission = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, indexDirs.size(), user == null ? null : user.getIdentity());
    lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
    final BooleanQuery lineageQuery = buildLineageQuery(flowFileUuids);
    final List<File> indexDirectories = directoryManager.getDirectories(startTimestamp, endTimestamp);
    if (indexDirectories.isEmpty()) {
        submission.getResult().update(Collections.emptyList(), 0L);
    } else {
        Collections.sort(indexDirectories, DirectoryUtils.OLDEST_INDEX_FIRST);
        for (final File indexDir : indexDirectories) {
            queryExecutor.submit(new QueryTask(lineageQuery, submission.getResult(), MAX_LINEAGE_NODES, indexManager, indexDir, eventStore, eventAuthorizer, EventTransformer.PLACEHOLDER_TRANSFORMER));
        }
    }
    // wait some short period of time for the computation to complete before returning the submission.
    try {
        submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS);
    } catch (final InterruptedException ie) {
        Thread.currentThread().interrupt();
    }
    return submission;
}

Example 4 with AsyncLineageSubmission

use of org.apache.nifi.provenance.AsyncLineageSubmission in project nifi by apache.

the class LuceneEventIndex method submitExpandChildren.

@Override
public ComputeLineageSubmission submitExpandChildren(final long eventId, final NiFiUser user, final EventAuthorizer authorizer) {
    final String userId = user == null ? null : user.getIdentity();
    try {
        final Optional<ProvenanceEventRecord> eventOption = eventStore.getEvent(eventId);
        if (!eventOption.isPresent()) {
            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
            lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
            submission.getResult().update(Collections.emptyList(), 0L);
            return submission;
        }
        final ProvenanceEventRecord event = eventOption.get();
        switch(event.getEventType()) {
            case CLONE:
            case FORK:
            case JOIN:
            case REPLAY:
                {
                    return submitLineageComputation(event.getChildUuids(), user, authorizer, LineageComputationType.EXPAND_CHILDREN, eventId, event.getEventTime(), Long.MAX_VALUE);
                }
            default:
                {
                    final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
                    lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                    submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded");
                    return submission;
                }
        }
    } catch (final Exception e) {
        final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId);
        lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
        submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e);
        return submission;
    }
}

Example 5 with AsyncLineageSubmission

use of org.apache.nifi.provenance.AsyncLineageSubmission in project nifi by apache.

the class LuceneEventIndex method submitExpandParents.

@Override
public ComputeLineageSubmission submitExpandParents(final long eventId, final NiFiUser user, final EventAuthorizer authorizer) {
    final String userId = user == null ? null : user.getIdentity();
    try {
        final Optional<ProvenanceEventRecord> eventOption = eventStore.getEvent(eventId);
        if (!eventOption.isPresent()) {
            final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
            lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
            submission.getResult().update(Collections.emptyList(), 0L);
            return submission;
        }
        final ProvenanceEventRecord event = eventOption.get();
        switch(event.getEventType()) {
            case JOIN:
            case FORK:
            case CLONE:
            case REPLAY:
                {
                    return submitLineageComputation(event.getParentUuids(), user, authorizer, LineageComputationType.EXPAND_PARENTS, eventId, event.getLineageStartDate(), event.getEventTime());
                }
            default:
                {
                    final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
                    lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
                    submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded");
                    return submission;
                }
        }
    } catch (final Exception e) {
        final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId);
        lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
        submission.getResult().setError("Failed to expand parents for lineage of event with ID " + eventId + " due to: " + e);
        return submission;
    }
}