Examples with Lineage org.apache.nifi.atlas.emulator.Lineage used on opensource projects

Example 11 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SSendComplete.

@Test
public void testS2SSendComplete() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SSend");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    testS2SSend(tc);
    final Lineage lineage = getLineage();
    // Complete path has hash.
    final Node pathC = lineage.findNode("nifi_flow_path", "Generate C, Remote Input Port", "c439cdca-e989-3491-0000-000000000000::1605753423@example");
    final Node pathT = lineage.findNode("nifi_flow_path", "GetTwitter, Remote Input Port", "b775b657-5a5b-3708-0000-000000000000::3843156947@example");
    // Generate C and GetTwitter have reported proper SEND lineage to the input port.
    final Node remoteInputPort = lineage.findNode("nifi_input_port", "input", "77919f59-533e-35a3");
    lineage.assertLink(pathC, remoteInputPort);
    lineage.assertLink(pathT, remoteInputPort);
    // nifi_data is created for each obscure input processor.
    final Node genC = lineage.findNode("nifi_data", "Generate C", "c439cdca-e989-3491");
    final Node genT = lineage.findNode("nifi_data", "GetTwitter", "b775b657-5a5b-3708");
    lineage.assertLink(genC, pathC);
    lineage.assertLink(genT, pathT);
}

Example 12 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testS2SGet.

/**
 * A client NiFi gets FlowFiles from a remote NiFi.
 */
@Test
public void testS2SGet() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SGet");
    final ProvenanceRecords prs = tc.provenanceRecords;
    // The remote port GUID is different than the Remote Output Ports.
    prs.add(pr("7375f8f6-4604-468d", "Remote Output Port", RECEIVE, "http://nifi.example.com:8080/nifi-api/data-transfer/output-ports" + "/392e7343-3950-329b-0000-000000000000/transactions/tx-1/flow-files"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2SGet", "S2SGet@example");
    final Node pathL = lineage.findNode("nifi_flow_path", "LogAttribute", "97cc5b27-22f3-3c3b");
    final Node pathP = lineage.findNode("nifi_flow_path", "PutFile", "4f3bfa4c-6427-3aac");
    final Node pathU = lineage.findNode("nifi_flow_path", "UpdateAttribute", "bb530e58-ee14-3cac");
    // These entities should be created by notification.
    final Node remoteOutputPortDataSet = lineage.findNode("nifi_output_port", "output", "392e7343-3950-329b");
    final Node remoteOutputPortProcess = lineage.findNode("nifi_flow_path", "Remote Output Port", "7375f8f6-4604-468d");
    final Node queueL = lineage.findNode("nifi_queue", "queue", "97cc5b27-22f3-3c3b");
    final Node queueP = lineage.findNode("nifi_queue", "queue", "4f3bfa4c-6427-3aac");
    final Node queueU = lineage.findNode("nifi_queue", "queue", "bb530e58-ee14-3cac");
    lineage.assertLink(remoteOutputPortDataSet, remoteOutputPortProcess);
    lineage.assertLink(flow, remoteOutputPortProcess);
    lineage.assertLink(remoteOutputPortProcess, queueL);
    lineage.assertLink(remoteOutputPortProcess, queueP);
    lineage.assertLink(remoteOutputPortProcess, queueU);
    lineage.assertLink(queueL, pathL);
    lineage.assertLink(queueP, pathP);
    lineage.assertLink(queueU, pathU);
}

Example 13 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testRemoteInvocation.

@Test
public void testRemoteInvocation() throws Exception {
    final TestConfiguration tc = new TestConfiguration("RemoteInvocation");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-23"));
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-24"));
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-25"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "RemoteInvocation", "RemoteInvocation@example");
    final Node path = lineage.findNode("nifi_flow_path", "DeleteHDFS", "2607ed95-c6ef-3636");
    final Node hdfsPath23 = lineage.findNode("hdfs_path", "/test/2017-10-23@example");
    final Node hdfsPath24 = lineage.findNode("hdfs_path", "/test/2017-10-24@example");
    final Node hdfsPath25 = lineage.findNode("hdfs_path", "/test/2017-10-25@example");
    lineage.assertLink(flow, path);
    lineage.assertLink(path, hdfsPath23);
    lineage.assertLink(path, hdfsPath24);
    lineage.assertLink(path, hdfsPath25);
}

Example 14 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testSimpleEventLevelCompletePath.

@Test
public void testSimpleEventLevelCompletePath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimpleEventLevel");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    String flowFileUUIDA = "A0000000-0000-0000";
    String flowFileUUIDB = "B0000000-0000-0000";
    prs.add(pr("d9257f7e-b78c-349a", "Generate A", CREATE, flowFileUUIDA));
    prs.add(pr("d84b9bdc-5e42-3b3b", "Generate B", CREATE, flowFileUUIDB));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/a.txt", flowFileUUIDA)));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/b.txt", flowFileUUIDB)));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDA));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDB));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    lineages.put(4L, createLineage(prs, 0, 2, 4));
    lineages.put(5L, createLineage(prs, 1, 3, 5));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "d9257f7e-b78c-349a");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A, PutFile, LogAttribute", "d9257f7e-b78c-349a-0000-000000000000::980416504@example");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B, PutFile, LogAttribute", "d84b9bdc-5e42-3b3b-0000-000000000000::442259660@example");
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outB = lineage.findNode("fs_path", "/tmp/nifi/b.txt@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genAPath, outA);
    lineage.assertLink(genBPath, outB);
}

Example 15 with Lineage

use of org.apache.nifi.atlas.emulator.Lineage in project nifi by apache.

the class ITReportLineageToAtlas method testSimpleEventLevelSimplePath.

@Test
public void testSimpleEventLevelSimplePath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimpleEventLevel");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("d9257f7e-b78c-349a", "Generate A", CREATE));
    prs.add(pr("d84b9bdc-5e42-3b3b", "Generate B", CREATE));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/a.txt")));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/b.txt")));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "d9257f7e-b78c-349a");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A", "d9257f7e-b78c-349a");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node queue = lineage.findNode("nifi_queue", "queue", "eaf013c1-aec5-39b0");
    final Node putFile = lineage.findNode("nifi_flow_path", "PutFile, LogAttribute", "eaf013c1-aec5-39b0");
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outB = lineage.findNode("fs_path", "/tmp/nifi/b.txt@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genAPath, queue);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genBPath, queue);
    lineage.assertLink(queue, putFile);
    lineage.assertLink(putFile, outA);
    lineage.assertLink(putFile, outB);
}