Examples with Lineage org.apache.nifi.atlas.emulator.Lineage used on opensource projects

Example 1 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testSimplestPath.

@Test
public void testSimplestPath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimplestFlowPath");
    test(tc);
    final Lineage lineage = getLineage();
    lineage.assertLink("nifi_flow", "SimplestFlowPath", "SimplestFlowPath@example", "nifi_flow_path", "GenerateFlowFile, LogAttribute", "d270e6f0-c5e0-38b9");
}

Example 2 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testRecordAndDataSetLevel.

@Test
public void testRecordAndDataSetLevel() throws Exception {
    final TestConfiguration tc = new TestConfiguration("RecordAndDataSetLevel");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    // Publish part
    final String ffIdA1 = "A1000000";
    final String ffIdB1 = "B1000000";
    // 0
    prs.add(pr("22be62d9-c4a1-3056", "GetFile", RECEIVE, "file:/tmp/input/A1.csv", ffIdA1));
    // 1
    prs.add(pr("22be62d9-c4a1-3056", "GetFile", RECEIVE, "file:/tmp/input/B1.csv", ffIdB1));
    // 2
    prs.add(pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/output/A1.csv", ffIdA1));
    // 3
    prs.add(pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/output/B1.csv", ffIdB1));
    // 4
    prs.add(pr("97641de3-fb76-3d95", "PublishKafkaRecord_0_10", SEND, "PLAINTEXT://localhost:9092/nifi-test", ffIdA1));
    // 5
    prs.add(pr("97641de3-fb76-3d95", "PublishKafkaRecord_0_10", SEND, "PLAINTEXT://localhost:9092/nifi-test", ffIdB1));
    // 6
    prs.add(pr("97641de3-fb76-3d95", "PublishKafkaRecord_0_10", DROP, ffIdA1));
    // 7
    prs.add(pr("97641de3-fb76-3d95", "PublishKafkaRecord_0_10", DROP, ffIdB1));
    // Consume part
    final String ffIdK1 = "K1000000";
    // Forked children
    final String ffIdA2 = "A2000000";
    // Forked children
    final String ffIdB2 = "B2000000";
    // 8
    prs.add(pr("529e6722-9b49-3b66", "ConsumeKafkaRecord_0_10", RECEIVE, "PLAINTEXT://localhost:9092/nifi-test", ffIdK1));
    // 9
    prs.add(pr("3f6d405e-6e3d-38c9", "PartitionRecord", FORK, ffIdK1));
    // 10
    prs.add(pr("db8bb12c-5cd3-3011", "UpdateAttribute", ATTRIBUTES_MODIFIED, ffIdA2));
    // 11
    prs.add(pr("db8bb12c-5cd3-3011", "UpdateAttribute", ATTRIBUTES_MODIFIED, ffIdB2));
    // 12
    prs.add(pr("062caf95-da40-3a57", "PutFile", SEND, "file:/tmp/consumed/A_20171101_100701.csv", ffIdA2));
    // 13
    prs.add(pr("062caf95-da40-3a57", "PutFile", SEND, "file:/tmp/consumed/B_20171101_100701.csv", ffIdB2));
    // 14
    prs.add(pr("062caf95-da40-3a57", "PutFile", DROP, ffIdA2));
    // 15
    prs.add(pr("062caf95-da40-3a57", "PutFile", DROP, ffIdB2));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    Map<Long, ComputeLineageResult> parents = tc.parentLineageResults;
    // Publish A1
    lineages.put(6L, createLineage(prs, 0, 2, 4, 6));
    // Publish B1
    lineages.put(7L, createLineage(prs, 1, 3, 5, 7));
    // Consumed and Forked K1
    parents.put(9L, createLineage(prs, 8, 9));
    // Processed A2
    lineages.put(14L, createLineage(prs, 9, 10, 12, 14));
    // Processed B2
    lineages.put(15L, createLineage(prs, 9, 11, 13, 15));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    // Publish part
    final Node inputFileA1 = lineage.findNode("fs_path", "/tmp/input/A1.csv@example");
    final Node inputFileB1 = lineage.findNode("fs_path", "/tmp/input/B1.csv@example");
    // These two flow paths are derived from the same set of Processors, but with different input files, and resulted different hashes.
    final Node getFileToPublishKafkaA = lineage.findNode("nifi_flow_path", "GetFile, PutFile, PublishKafkaRecord_0_10", "22be62d9-c4a1-3056-0000-000000000000::2823953997@example");
    final Node getFileToPublishKafkaB = lineage.findNode("nifi_flow_path", "GetFile, PutFile, PublishKafkaRecord_0_10", "22be62d9-c4a1-3056-0000-000000000000::568010061@example");
    lineage.assertLink(inputFileA1, getFileToPublishKafkaA);
    lineage.assertLink(inputFileB1, getFileToPublishKafkaB);
    final Node nifiTestTopic = lineage.findNode("kafka_topic", "nifi-test@example");
    final Node outputFileA = lineage.findNode("fs_path", "/tmp/output/A1.csv@example");
    final Node outputFileB = lineage.findNode("fs_path", "/tmp/output/B1.csv@example");
    lineage.assertLink(getFileToPublishKafkaA, nifiTestTopic);
    lineage.assertLink(getFileToPublishKafkaB, nifiTestTopic);
    lineage.assertLink(getFileToPublishKafkaA, outputFileA);
    lineage.assertLink(getFileToPublishKafkaB, outputFileB);
    // Consume part
    final Node consumeNifiTestTopic = lineage.findNode("nifi_flow_path", "ConsumeKafkaRecord_0_10", "529e6722-9b49-3b66-0000-000000000000::3649132843@example");
    final Node forkedA = lineage.findNode("nifi_queue", "FORK", "3f6d405e-6e3d-38c9-0000-000000000000::234149075@example");
    final Node forkedB = lineage.findNode("nifi_queue", "FORK", "3f6d405e-6e3d-38c9-0000-000000000000::2377021542@example");
    lineage.assertLink(consumeNifiTestTopic, forkedA);
    lineage.assertLink(consumeNifiTestTopic, forkedB);
    final Node partitionToPutA = lineage.findNode("nifi_flow_path", "PartitionRecord, UpdateAttribute, PutFile", "3f6d405e-6e3d-38c9-0000-000000000000::234149075@example");
    final Node partitionToPutB = lineage.findNode("nifi_flow_path", "PartitionRecord, UpdateAttribute, PutFile", "3f6d405e-6e3d-38c9-0000-000000000000::2377021542@example");
    final Node consumedFileA = lineage.findNode("fs_path", "/tmp/consumed/A_20171101_100701.csv@example");
    final Node consumedFileB = lineage.findNode("fs_path", "/tmp/consumed/B_20171101_100701.csv@example");
    lineage.assertLink(forkedA, partitionToPutA);
    lineage.assertLink(forkedB, partitionToPutB);
    lineage.assertLink(partitionToPutA, consumedFileA);
    lineage.assertLink(partitionToPutB, consumedFileB);
}

Example 3 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SReceive.

/**
 * A remote NiFi receives FlowFiles from remote client NiFis.
 * This NiFi instance owns RootProcessGroup input port.
 */
@Test
public void testS2SReceive() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SReceive");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("77919f59-533e-35a3", "Input Port", RECEIVE, "http://nifi.example.com:8080/nifi-api/data-transfer/output-ports" + "/77919f59-533e-35a3-0000-000000000000/transactions/tx-1/flow-files"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2SReceive", "S2SReceive@example");
    final Node path = lineage.findNode("nifi_flow_path", "input, UpdateAttribute", "77919f59-533e-35a3");
    final Node inputPort = lineage.findNode("nifi_input_port", "input", "77919f59-533e-35a3");
    lineage.assertLink(flow, path);
    lineage.assertLink(flow, inputPort);
    lineage.assertLink(inputPort, path);
}

Example 4 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2STransfer.

/**
 * A remote NiFi transfers FlowFiles to remote client NiFis.
 * This NiFi instance owns RootProcessGroup output port.
 */
@Test
public void testS2STransfer() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2STransfer");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("392e7343-3950-329b", "Output Port", SEND, "http://nifi.example.com:8080/nifi-api/data-transfer/output-ports" + "/392e7343-3950-329b-0000-000000000000/transactions/tx-1/flow-files"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2STransfer", "S2STransfer@example");
    final Node path = lineage.findNode("nifi_flow_path", "GenerateFlowFile, output", "1b9f81db-a0fd-389a");
    final Node outputPort = lineage.findNode("nifi_output_port", "output", "392e7343-3950-329b");
    lineage.assertLink(flow, path);
    lineage.assertLink(path, outputPort);
}

Example 5 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SSend.

/**
 * A client NiFi sends FlowFiles to a remote NiFi.
 */
private void testS2SSend(TestConfiguration tc) throws Exception {
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("ca71e4d9-2a4f-3970", "Generate A", CREATE));
    prs.add(pr("c439cdca-e989-3491", "Generate C", CREATE));
    prs.add(pr("b775b657-5a5b-3708", "GetTwitter", CREATE));
    // The remote port GUID is different than the Remote Input Ports.
    prs.add(pr("f31a6b53-3077-4c59", "Remote Input Port", SEND, "http://nifi.example.com:8080/nifi-api/data-transfer/input-ports" + "/77919f59-533e-35a3-0000-000000000000/transactions/tx-1/flow-files"));
    prs.add(pr("f31a6b53-3077-4c59", "Remote Input Port", SEND, "http://nifi.example.com:8080/nifi-api/data-transfer/input-ports" + "/77919f59-533e-35a3-0000-000000000000/transactions/tx-2/flow-files"));
    // C
    prs.add(pr("f31a6b53-3077-4c59", "Remote Input Port", DROP));
    // Twitter
    prs.add(pr("f31a6b53-3077-4c59", "Remote Input Port", DROP));
    // Generate C created a FlowFile, then it's sent via S2S
    tc.addLineage(createLineage(prs, 1, 3, 5));
    // GetTwitter created a FlowFile, then it's sent via S2S
    tc.addLineage(createLineage(prs, 2, 4, 6));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2SSend", "S2SSend@example");
    final Node pathA = lineage.findNode("nifi_flow_path", "Generate A", "ca71e4d9-2a4f-3970");
    final Node pathB = lineage.findNode("nifi_flow_path", "Generate B", "333255b6-eb02-3056");
    final Node pathC = lineage.findNode("nifi_flow_path", "Generate C", "c439cdca-e989-3491");
    final Node pathT = lineage.findNode("nifi_flow_path", "GetTwitter", "b775b657-5a5b-3708");
    final Node pathI = lineage.findNode("nifi_flow_path", "InactiveProcessor", "7033f311-ac68-3cab");
    // UpdateAttribute has multiple incoming paths, so it generates a queue to receive those.
    final Node queueU = lineage.findNode("nifi_queue", "queue", "c5392447-e9f1-33ad");
    final Node pathU = lineage.findNode("nifi_flow_path", "UpdateAttribute", "c5392447-e9f1-33ad");
    // These are starting paths.
    lineage.assertLink(flow, pathA);
    lineage.assertLink(flow, pathB);
    lineage.assertLink(flow, pathC);
    lineage.assertLink(flow, pathT);
    lineage.assertLink(flow, pathI);
    // Multiple paths connected to the same path.
    lineage.assertLink(pathB, queueU);
    lineage.assertLink(pathC, queueU);
    lineage.assertLink(queueU, pathU);
}