Examples with Lineage org.apache.nifi.atlas.emulator.Lineage used on opensource projects

Example 6 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SReceiveAndSendCombination.

@Test
public void testS2SReceiveAndSendCombination() throws Exception {
    testS2SReceive();
    testS2SSendSimple();
    final Lineage lineage = getLineage();
    final Node remoteFlow = lineage.findNode("nifi_flow", "S2SReceive", "S2SReceive@example");
    final Node localFlow = lineage.findNode("nifi_flow", "S2SSend", "S2SSend@example");
    final Node remoteInputPortQ = lineage.findNode("nifi_queue", "queue", "f31a6b53-3077-4c59");
    final Node remoteInputPortP = lineage.findNode("nifi_flow_path", "Remote Input Port", "f31a6b53-3077-4c59");
    final Node inputPort = lineage.findNode("nifi_input_port", "input", "77919f59-533e-35a3");
    final Node pathC = lineage.findNode("nifi_flow_path", "Generate C", "c439cdca-e989-3491");
    final Node pathT = lineage.findNode("nifi_flow_path", "GetTwitter", "b775b657-5a5b-3708");
    // Remote flow owns the inputPort.
    lineage.assertLink(remoteFlow, inputPort);
    // These paths within local flow sends data to the remote flow through the remote input port.
    lineage.assertLink(localFlow, pathC);
    lineage.assertLink(localFlow, pathT);
    lineage.assertLink(pathC, remoteInputPortQ);
    lineage.assertLink(pathT, remoteInputPortQ);
    lineage.assertLink(remoteInputPortQ, remoteInputPortP);
    lineage.assertLink(remoteInputPortP, inputPort);
}

Example 7 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testSingleFlowPath.

@Test
public void testSingleFlowPath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SingleFlowPath");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("2e9a2852-228f-379b", "ConsumeKafka_0_11", RECEIVE, "PLAINTEXT://0.kafka.example.com:6667/topic-a"));
    prs.add(pr("5a56149a-d82a-3242", "PublishKafka_0_11", SEND, "PLAINTEXT://0.kafka.example.com:6667/topic-b"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "SingleFlowPath", "SingleFlowPath@example");
    final Node path = lineage.findNode("nifi_flow_path", "ConsumeKafka_0_11, UpdateAttribute, ConvertJSONToSQL, PutSQL, PublishKafka_0_11", "2e9a2852-228f-379b");
    final Node topicA = lineage.findNode("kafka_topic", "topic-a@example");
    final Node topicB = lineage.findNode("kafka_topic", "topic-b@example");
    lineage.assertLink(flow, path);
    lineage.assertLink(topicA, path);
    lineage.assertLink(path, topicB);
}

Example 8 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SDirect.

/**
 * A client NiFi gets FlowFiles from a remote output port and sends it to a remote input port without doing anything.
 */
@Test
public void testS2SDirect() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SDirect");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("d73d9115-b987-4ffc", "Remote Output Port", RECEIVE, "http://nifi.example.com:8080/nifi-api/data-transfer/output-ports" + "/015f1040-dcd7-17bd-5c1f-e31afe0a09a4/transactions/tx-1/flow-files"));
    prs.add((pr("a4f14247-89aa-4e6c", "Remote Input Port", SEND, "http://nifi.example.com:8080/nifi-api/data-transfer/input-ports" + "/015f101e-dcd7-17bd-8899-1a723733521a/transactions/tx-2/flow-files")));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    // Received from remote output port, then sent it via remote input port
    lineages.put(1L, createLineage(prs, 0, 1));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2SDirect", "S2SDirect@example");
    final Node remoteOutputPort = lineage.findNode("nifi_output_port", "output", "015f1040-dcd7-17bd-5c1f-e31afe0a09a4@example");
    final Node remoteOutputPortP = lineage.findNode("nifi_flow_path", "Remote Output Port", "d73d9115-b987-4ffc");
    final Node remoteInputPortQ = lineage.findNode("nifi_queue", "queue", "a4f14247-89aa-4e6c");
    final Node remoteInputPortP = lineage.findNode("nifi_flow_path", "Remote Input Port", "a4f14247-89aa-4e6c");
    final Node remoteInputPort = lineage.findNode("nifi_input_port", "input", "015f101e-dcd7-17bd-8899-1a723733521a@example");
    // Even if there is no Processor, lineage can be reported using root flow_path.
    lineage.assertLink(flow, remoteOutputPortP);
    lineage.assertLink(remoteOutputPort, remoteOutputPortP);
    lineage.assertLink(remoteOutputPortP, remoteInputPortQ);
    lineage.assertLink(remoteInputPortQ, remoteInputPortP);
    lineage.assertLink(remoteInputPortP, remoteInputPort);
}

Example 9 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testMergedEvents.

@Test
public void testMergedEvents() throws Exception {
    final TestConfiguration tc = new TestConfiguration("MergedEvents");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    final String flowFileUUIDA = "A0000000-0000-0000";
    final String flowFileUUIDB = "B0000000-0000-0000";
    final String flowFileUUIDC = "C0000000-0000-0000";
    final String flowFileUUIDD = "D0000000-0000-0000";
    // Merged B and C.
    final String flowFileUUIDBC = "BC000000-0000-0000";
    // 0
    prs.add(pr("f585d83b-2a03-37cf", "Generate A", CREATE, flowFileUUIDA));
    // 1
    prs.add(pr("59a7c1f9-9a73-3cc6", "Generate B", CREATE, flowFileUUIDB));
    // 2
    prs.add(pr("d6c3f282-e03d-316c", "Generate C", CREATE, flowFileUUIDC));
    // 3
    prs.add(pr("f9593a5a-f0d5-3e87", "Generate D", CREATE, flowFileUUIDD));
    // Original files are dropped.
    // 4
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", JOIN, flowFileUUIDBC));
    // 5
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", DROP, flowFileUUIDB));
    // 6
    prs.add(pr("c77dd033-bb9e-39ea", "MergeContent", DROP, flowFileUUIDC));
    // 7
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/a.txt", flowFileUUIDA)));
    // 8
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/bc.txt", flowFileUUIDBC)));
    // 9
    prs.add((pr("93f8ad14-6ee6-34c1", "PutFile", SEND, "file:/tmp/nifi/d.txt", flowFileUUIDD)));
    // 10
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDA));
    // 11
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDBC));
    // 12
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDD));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    final ComputeLineageResult lineageB = createLineage(prs, 1, 4, 5);
    final ComputeLineageResult lineageC = createLineage(prs, 2, 4, 6);
    // B
    lineages.put(5L, lineageB);
    // C
    lineages.put(6L, lineageC);
    // A
    lineages.put(10L, createLineage(prs, 0, 7, 10));
    // BC
    lineages.put(11L, createLineage(prs, 4, 8, 11));
    // D
    lineages.put(12L, createLineage(prs, 3, 9, 12));
    Map<Long, ComputeLineageResult> parents = tc.parentLineageResults;
    parents.put(4L, compositeLineages(lineageB, lineageC));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "f585d83b-2a03-37cf");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "59a7c1f9-9a73-3cc6");
    final Node genC = lineage.findNode("nifi_data", "Generate C", "d6c3f282-e03d-316c");
    final Node genD = lineage.findNode("nifi_data", "Generate D", "f9593a5a-f0d5-3e87");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A, PutFile, LogAttribute", "f585d83b-2a03-37cf-0000-000000000000::1003499964@example");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B", "59a7c1f9-9a73-3cc6-0000-000000000000::45412830@example");
    final Node genCPath = lineage.findNode("nifi_flow_path", "Generate C", "d6c3f282-e03d-316c-0000-000000000000::1968410985@example");
    final Node genDPath = lineage.findNode("nifi_flow_path", "Generate D, PutFile, LogAttribute", "f9593a5a-f0d5-3e87-0000-000000000000::4257576567@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genC, genCPath);
    lineage.assertLink(genD, genDPath);
    // B and C were merged together, while A and D were processed individually.
    final Node joinBC = lineage.findNode("nifi_queue", "JOIN", "c77dd033-bb9e-39ea-0000-000000000000::2370367315@example");
    final Node bcPath = lineage.findNode("nifi_flow_path", "MergeContent, PutFile, LogAttribute", "c77dd033-bb9e-39ea-0000-000000000000::2370367315@example");
    lineage.assertLink(genBPath, joinBC);
    lineage.assertLink(genCPath, joinBC);
    lineage.assertLink(joinBC, bcPath);
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outBC = lineage.findNode("fs_path", "/tmp/nifi/bc.txt@example");
    final Node outD = lineage.findNode("fs_path", "/tmp/nifi/d.txt@example");
    lineage.assertLink(genAPath, outA);
    lineage.assertLink(bcPath, outBC);
    lineage.assertLink(genDPath, outD);
}

Example 10 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SSendSimple.

@Test
public void testS2SSendSimple() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SSend");
    testS2SSend(tc);
    final Lineage lineage = getLineage();
    // The FlowFile created by Generate A has not been finished (by DROP event, but SIMPLE_PATH strategy can report it.
    final Node pathA = lineage.findNode("nifi_flow_path", "Generate A", "ca71e4d9-2a4f-3970");
    final Node genA = lineage.findNode("nifi_data", "Generate A", "ca71e4d9-2a4f-3970");
    lineage.assertLink(genA, pathA);
    final Node pathC = lineage.findNode("nifi_flow_path", "Generate C", "c439cdca-e989-3491");
    final Node pathT = lineage.findNode("nifi_flow_path", "GetTwitter", "b775b657-5a5b-3708");
    // Generate C and GetTwitter have reported proper SEND lineage to the input port.
    final Node remoteInputPortD = lineage.findNode("nifi_input_port", "input", "77919f59-533e-35a3");
    final Node remoteInputPortP = lineage.findNode("nifi_flow_path", "Remote Input Port", "f31a6b53-3077-4c59");
    final Node remoteInputPortQ = lineage.findNode("nifi_queue", "queue", "f31a6b53-3077-4c59");
    lineage.assertLink(pathC, remoteInputPortQ);
    lineage.assertLink(pathT, remoteInputPortQ);
    lineage.assertLink(remoteInputPortQ, remoteInputPortP);
    lineage.assertLink(remoteInputPortP, remoteInputPortD);
    // nifi_data is created for each obscure input processor.
    final Node genC = lineage.findNode("nifi_data", "Generate C", "c439cdca-e989-3491");
    final Node genT = lineage.findNode("nifi_data", "GetTwitter", "b775b657-5a5b-3708");
    lineage.assertLink(genC, pathC);
    lineage.assertLink(genT, pathT);
}