Examples with ComputeLineageResult org.apache.nifi.provenance.lineage.ComputeLineageResult used on opensource projects

Example 6 with ComputeLineageResult

use of org.apache.nifi.provenance.lineage.ComputeLineageResult in project nifi by apache.

the class CompleteFlowPathLineage method processEvent.

@Override
public void processEvent(AnalysisContext analysisContext, NiFiFlow nifiFlow, ProvenanceEventRecord event) {
    if (!ProvenanceEventType.DROP.equals(event.getEventType())) {
        return;
    }
    final ComputeLineageResult lineage = analysisContext.queryLineage(event.getEventId());
    // Construct a tree model to traverse backwards.
    final Map<String, List<LineageNode>> lineageTree = new HashMap<>();
    analyzeLineageTree(lineage, lineageTree);
    final LineagePath lineagePath = new LineagePath();
    extractLineagePaths(analysisContext, lineageTree, lineagePath, event);
    analyzeLineagePath(analysisContext, lineagePath);
    // Input and output data set are both required to report lineage.
    List<Tuple<NiFiFlowPath, DataSetRefs>> createdFlowPaths = new ArrayList<>();
    if (lineagePath.isComplete()) {
        createCompleteFlowPath(nifiFlow, lineagePath, createdFlowPaths);
        for (Tuple<NiFiFlowPath, DataSetRefs> createdFlowPath : createdFlowPaths) {
            final NiFiFlowPath flowPath = createdFlowPath.getKey();
            createEntity(toReferenceable(flowPath, nifiFlow));
            addDataSetRefs(nifiFlow, Collections.singleton(flowPath), createdFlowPath.getValue());
        }
        createdFlowPaths.clear();
    }
}

Example 7 with ComputeLineageResult

use of org.apache.nifi.provenance.lineage.ComputeLineageResult in project nifi by apache.

the class ITReportLineageToAtlas method testMergedEvents.

@Test
public void testMergedEvents() throws Exception {
    final TestConfiguration tc = new TestConfiguration("MergedEvents");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    final String flowFileUUIDA = "A0000000-0000-0000";
    final String flowFileUUIDB = "B0000000-0000-0000";
    final String flowFileUUIDC = "C0000000-0000-0000";
    final String flowFileUUIDD = "D0000000-0000-0000";
    // Merged B and C.
    final String flowFileUUIDBC = "BC000000-0000-0000";
    // 0
    prs.add(pr("f585d83b-2a03-37cf", "Generate A", CREATE, flowFileUUIDA));
    // 1
    prs.add(pr("59a7c1f9-9a73-3cc6", "Generate B", CREATE, flowFileUUIDB));
    // 2
    prs.add(pr("d6c3f282-e03d-316c", "Generate C", CREATE, flowFileUUIDC));
    // 3
    prs.add(pr("f9593a5a-f0d5-3e87", "Generate D", CREATE, flowFileUUIDD));
    // Original files are dropped.
    // 4
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", JOIN, flowFileUUIDBC));
    // 5
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", DROP, flowFileUUIDB));
    // 6
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", DROP, flowFileUUIDC));
    // 7
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/a.txt", flowFileUUIDA)));
    // 8
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/bc.txt", flowFileUUIDBC)));
    // 9
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/d.txt", flowFileUUIDD)));
    // 10
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDA));
    // 11
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDBC));
    // 12
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDD));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    final ComputeLineageResult lineageB = createLineage(prs, 1, 4, 5);
    final ComputeLineageResult lineageC = createLineage(prs, 2, 4, 6);
    // B
    lineages.put(5L, lineageB);
    // C
    lineages.put(6L, lineageC);
    // A
    lineages.put(10L, createLineage(prs, 0, 7, 10));
    // BC
    lineages.put(11L, createLineage(prs, 4, 8, 11));
    // D
    lineages.put(12L, createLineage(prs, 3, 9, 12));
    Map<Long, ComputeLineageResult> parents = tc.parentLineageResults;
    parents.put(4L, compositeLineages(lineageB, lineageC));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "f585d83b-2a03-37cf");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "59a7c1f9-9a73-3cc6");
    final Node genC = lineage.findNode("nifi_data", "Generate C", "d6c3f282-e03d-316c");
    final Node genD = lineage.findNode("nifi_data", "Generate D", "f9593a5a-f0d5-3e87");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A, PutFile, LogAttribute", "f585d83b-2a03-37cf-0000-000000000000::1003499964@example");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B", "59a7c1f9-9a73-3cc6-0000-000000000000::45412830@example");
    final Node genCPath = lineage.findNode("nifi_flow_path", "Generate C", "d6c3f282-e03d-316c-0000-000000000000::1968410985@example");
    final Node genDPath = lineage.findNode("nifi_flow_path", "Generate D, PutFile, LogAttribute", "f9593a5a-f0d5-3e87-0000-000000000000::4257576567@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genC, genCPath);
    lineage.assertLink(genD, genDPath);
    // B and C were merged together, while A and D were processed individually.
    final Node joinBC = lineage.findNode("nifi_queue", "JOIN", "c77dd033-bb9e-39ea-0000-000000000000::2370367315@example");
    final Node bcPath = lineage.findNode("nifi_flow_path", "MergeContent, PutFile, LogAttribute", "c77dd033-bb9e-39ea-0000-000000000000::2370367315@example");
    lineage.assertLink(genBPath, joinBC);
    lineage.assertLink(genCPath, joinBC);
    lineage.assertLink(joinBC, bcPath);
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outBC = lineage.findNode("fs_path", "/tmp/nifi/bc.txt@example");
    final Node outD = lineage.findNode("fs_path", "/tmp/nifi/d.txt@example");
    lineage.assertLink(genAPath, outA);
    lineage.assertLink(bcPath, outBC);
    lineage.assertLink(genDPath, outD);
}

Example 8 with ComputeLineageResult

use of org.apache.nifi.provenance.lineage.ComputeLineageResult in project nifi by apache.

the class ITReportLineageToAtlas method testSimpleEventLevelCompletePath.

@Test
public void testSimpleEventLevelCompletePath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimpleEventLevel");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    String flowFileUUIDA = "A0000000-0000-0000";
    String flowFileUUIDB = "B0000000-0000-0000";
    prs.add(pr("d9257f7e-b78c-349a", "Generate A", CREATE, flowFileUUIDA));
    prs.add(pr("d84b9bdc-5e42-3b3b", "Generate B", CREATE, flowFileUUIDB));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/a.txt", flowFileUUIDA)));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/b.txt", flowFileUUIDB)));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDA));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDB));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    lineages.put(4L, createLineage(prs, 0, 2, 4));
    lineages.put(5L, createLineage(prs, 1, 3, 5));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "d9257f7e-b78c-349a");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A, PutFile, LogAttribute", "d9257f7e-b78c-349a-0000-000000000000::980416504@example");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B, PutFile, LogAttribute", "d84b9bdc-5e42-3b3b-0000-000000000000::442259660@example");
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outB = lineage.findNode("fs_path", "/tmp/nifi/b.txt@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genAPath, outA);
    lineage.assertLink(genBPath, outB);
}

Example 9 with ComputeLineageResult

use of org.apache.nifi.provenance.lineage.ComputeLineageResult in project nifi by apache.

the class ITReportLineageToAtlas method compositeLineages.

private ComputeLineageResult compositeLineages(ComputeLineageResult... results) throws InterruptedException {
    final ComputeLineageResult lineage = mock(ComputeLineageResult.class);
    when(lineage.awaitCompletion(anyLong(), any())).thenReturn(true);
    final List<LineageEdge> edges = new ArrayList<>();
    final Set<LineageNode> nodes = new LinkedHashSet<>();
    for (int i = 0; i < results.length; i++) {
        edges.addAll(results[i].getEdges());
        nodes.addAll(results[i].getNodes());
    }
    when(lineage.getEdges()).thenReturn(edges);
    when(lineage.getNodes()).thenReturn(new ArrayList<>(nodes));
    return lineage;
}

Example 10 with ComputeLineageResult

use of org.apache.nifi.provenance.lineage.ComputeLineageResult in project nifi by apache.

the class DtoFactory method createLineageDto.

/**
 * Creates a LineageDTO for the specified Lineage.
 *
 * @param computeLineageSubmission submission
 * @return dto
 */
public LineageDTO createLineageDto(final ComputeLineageSubmission computeLineageSubmission) {
    // build the lineage dto
    final LineageDTO dto = new LineageDTO();
    final LineageRequestDTO requestDto = new LineageRequestDTO();
    final LineageResultsDTO resultsDto = new LineageResultsDTO();
    // include the original request and results
    dto.setRequest(requestDto);
    dto.setResults(resultsDto);
    // rebuild the request from the submission object
    switch(computeLineageSubmission.getLineageComputationType()) {
        case EXPAND_CHILDREN:
            requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
            requestDto.setLineageRequestType(LineageRequestType.CHILDREN);
            break;
        case EXPAND_PARENTS:
            requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
            requestDto.setLineageRequestType(LineageRequestType.PARENTS);
            break;
        case FLOWFILE_LINEAGE:
            final Collection<String> uuids = computeLineageSubmission.getLineageFlowFileUuids();
            if (uuids.size() == 1) {
                requestDto.setUuid(uuids.iterator().next());
            }
            requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
            requestDto.setLineageRequestType(LineageRequestType.FLOWFILE);
            break;
    }
    // include lineage details
    dto.setId(computeLineageSubmission.getLineageIdentifier());
    dto.setSubmissionTime(computeLineageSubmission.getSubmissionTime());
    // create the results dto
    final ComputeLineageResult results = computeLineageSubmission.getResult();
    dto.setFinished(results.isFinished());
    dto.setPercentCompleted(results.getPercentComplete());
    dto.setExpiration(results.getExpiration());
    final List<LineageNode> nodes = results.getNodes();
    final List<LineageEdge> edges = results.getEdges();
    final List<ProvenanceNodeDTO> nodeDtos = new ArrayList<>();
    if (results.isFinished()) {
        // create the node dto's
        for (final LineageNode node : nodes) {
            switch(node.getNodeType()) {
                case FLOWFILE_NODE:
                    nodeDtos.add(createFlowFileNodeDTO(node));
                    break;
                case PROVENANCE_EVENT_NODE:
                    nodeDtos.add(createProvenanceEventNodeDTO((ProvenanceEventLineageNode) node));
                    break;
            }
        }
    }
    resultsDto.setNodes(nodeDtos);
    // include any errors
    if (results.getError() != null) {
        final Set<String> errors = new HashSet<>();
        errors.add(results.getError());
        resultsDto.setErrors(errors);
    }
    // create the link dto's
    final List<ProvenanceLinkDTO> linkDtos = new ArrayList<>();
    for (final LineageEdge edge : edges) {
        linkDtos.add(createProvenanceLinkDTO(edge));
    }
    resultsDto.setLinks(linkDtos);
    return dto;
}