Examples with Node org.apache.nifi.atlas.emulator.Node used on opensource projects

Example 11 with Node

use of org.apache.nifi.atlas.emulator.Node in project nifi by apache.

the class ITReportLineageToAtlas method testS2SGet.

/**
 * A client NiFi gets FlowFiles from a remote NiFi.
 */
@Test
public void testS2SGet() throws Exception {
    final TestConfiguration tc = new TestConfiguration("S2SGet");
    final ProvenanceRecords prs = tc.provenanceRecords;
    // The remote port GUID is different than the Remote Output Ports.
    prs.add(pr("7375f8f6-4604-468d", "Remote Output Port", RECEIVE, "http://nifi.example.com:8080/nifi-api/data-transfer/output-ports" + "/392e7343-3950-329b-0000-000000000000/transactions/tx-1/flow-files"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "S2SGet", "S2SGet@example");
    final Node pathL = lineage.findNode("nifi_flow_path", "LogAttribute", "97cc5b27-22f3-3c3b");
    final Node pathP = lineage.findNode("nifi_flow_path", "PutFile", "4f3bfa4c-6427-3aac");
    final Node pathU = lineage.findNode("nifi_flow_path", "UpdateAttribute", "bb530e58-ee14-3cac");
    // These entities should be created by notification.
    final Node remoteOutputPortDataSet = lineage.findNode("nifi_output_port", "output", "392e7343-3950-329b");
    final Node remoteOutputPortProcess = lineage.findNode("nifi_flow_path", "Remote Output Port", "7375f8f6-4604-468d");
    final Node queueL = lineage.findNode("nifi_queue", "queue", "97cc5b27-22f3-3c3b");
    final Node queueP = lineage.findNode("nifi_queue", "queue", "4f3bfa4c-6427-3aac");
    final Node queueU = lineage.findNode("nifi_queue", "queue", "bb530e58-ee14-3cac");
    lineage.assertLink(remoteOutputPortDataSet, remoteOutputPortProcess);
    lineage.assertLink(flow, remoteOutputPortProcess);
    lineage.assertLink(remoteOutputPortProcess, queueL);
    lineage.assertLink(remoteOutputPortProcess, queueP);
    lineage.assertLink(remoteOutputPortProcess, queueU);
    lineage.assertLink(queueL, pathL);
    lineage.assertLink(queueP, pathP);
    lineage.assertLink(queueU, pathU);
}

Example 12 with Node

use of org.apache.nifi.atlas.emulator.Node in project nifi by apache.

the class ITReportLineageToAtlas method testRemoteInvocation.

@Test
public void testRemoteInvocation() throws Exception {
    final TestConfiguration tc = new TestConfiguration("RemoteInvocation");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-23"));
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-24"));
    prs.add(pr("2607ed95-c6ef-3636", "DeleteHDFS", REMOTE_INVOCATION, "hdfs://nn1.example.com:8020/test/2017-10-25"));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node flow = lineage.findNode("nifi_flow", "RemoteInvocation", "RemoteInvocation@example");
    final Node path = lineage.findNode("nifi_flow_path", "DeleteHDFS", "2607ed95-c6ef-3636");
    final Node hdfsPath23 = lineage.findNode("hdfs_path", "/test/2017-10-23@example");
    final Node hdfsPath24 = lineage.findNode("hdfs_path", "/test/2017-10-24@example");
    final Node hdfsPath25 = lineage.findNode("hdfs_path", "/test/2017-10-25@example");
    lineage.assertLink(flow, path);
    lineage.assertLink(path, hdfsPath23);
    lineage.assertLink(path, hdfsPath24);
    lineage.assertLink(path, hdfsPath25);
}

Example 13 with Node

use of org.apache.nifi.atlas.emulator.Node in project nifi by apache.

the class ITReportLineageToAtlas method testSimpleEventLevelCompletePath.

@Test
public void testSimpleEventLevelCompletePath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimpleEventLevel");
    tc.properties.put(NIFI_LINEAGE_STRATEGY, LINEAGE_STRATEGY_COMPLETE_PATH.getValue());
    final ProvenanceRecords prs = tc.provenanceRecords;
    String flowFileUUIDA = "A0000000-0000-0000";
    String flowFileUUIDB = "B0000000-0000-0000";
    prs.add(pr("d9257f7e-b78c-349a", "Generate A", CREATE, flowFileUUIDA));
    prs.add(pr("d84b9bdc-5e42-3b3b", "Generate B", CREATE, flowFileUUIDB));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/a.txt", flowFileUUIDA)));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/b.txt", flowFileUUIDB)));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDA));
    prs.add(pr("bfc30bc3-48cf-332a", "LogAttribute", DROP, flowFileUUIDB));
    Map<Long, ComputeLineageResult> lineages = tc.lineageResults;
    lineages.put(4L, createLineage(prs, 0, 2, 4));
    lineages.put(5L, createLineage(prs, 1, 3, 5));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "d9257f7e-b78c-349a");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A, PutFile, LogAttribute", "d9257f7e-b78c-349a-0000-000000000000::980416504@example");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B, PutFile, LogAttribute", "d84b9bdc-5e42-3b3b-0000-000000000000::442259660@example");
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outB = lineage.findNode("fs_path", "/tmp/nifi/b.txt@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genAPath, outA);
    lineage.assertLink(genBPath, outB);
}

Example 14 with Node

use of org.apache.nifi.atlas.emulator.Node in project nifi by apache.

the class ITReportLineageToAtlas method testSimpleEventLevelSimplePath.

@Test
public void testSimpleEventLevelSimplePath() throws Exception {
    final TestConfiguration tc = new TestConfiguration("SimpleEventLevel");
    final ProvenanceRecords prs = tc.provenanceRecords;
    prs.add(pr("d9257f7e-b78c-349a", "Generate A", CREATE));
    prs.add(pr("d84b9bdc-5e42-3b3b", "Generate B", CREATE));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/a.txt")));
    prs.add((pr("eaf013c1-aec5-39b0", "PutFile", SEND, "file:/tmp/nifi/b.txt")));
    test(tc);
    waitNotificationsGetDelivered();
    final Lineage lineage = getLineage();
    final Node genA = lineage.findNode("nifi_data", "Generate A", "d9257f7e-b78c-349a");
    final Node genB = lineage.findNode("nifi_data", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node genAPath = lineage.findNode("nifi_flow_path", "Generate A", "d9257f7e-b78c-349a");
    final Node genBPath = lineage.findNode("nifi_flow_path", "Generate B", "d84b9bdc-5e42-3b3b");
    final Node queue = lineage.findNode("nifi_queue", "queue", "eaf013c1-aec5-39b0");
    final Node putFile = lineage.findNode("nifi_flow_path", "PutFile, LogAttribute", "eaf013c1-aec5-39b0");
    final Node outA = lineage.findNode("fs_path", "/tmp/nifi/a.txt@example");
    final Node outB = lineage.findNode("fs_path", "/tmp/nifi/b.txt@example");
    lineage.assertLink(genA, genAPath);
    lineage.assertLink(genAPath, queue);
    lineage.assertLink(genB, genBPath);
    lineage.assertLink(genBPath, queue);
    lineage.assertLink(queue, putFile);
    lineage.assertLink(putFile, outA);
    lineage.assertLink(putFile, outB);
}

Example 15 with Node

use of org.apache.nifi.atlas.emulator.Node in project nifi by apache.

the class ITReportLineageToAtlas method testS2STransferAndGetCombination.

@Test
public void testS2STransferAndGetCombination() throws Exception {
    testS2STransfer();
    testS2SGet();
    final Lineage lineage = getLineage();
    final Node remoteFlow = lineage.findNode("nifi_flow", "S2STransfer", "S2STransfer@example");
    final Node localFlow = lineage.findNode("nifi_flow", "S2SGet", "S2SGet@example");
    final Node remoteGen = lineage.findNode("nifi_flow_path", "GenerateFlowFile, output", "1b9f81db-a0fd-389a");
    final Node outputPort = lineage.findNode("nifi_output_port", "output", "392e7343-3950-329b");
    final Node remoteOutputPortP = lineage.findNode("nifi_flow_path", "Remote Output Port", "7375f8f6-4604-468d");
    final Node queueL = lineage.findNode("nifi_queue", "queue", "97cc5b27-22f3-3c3b");
    final Node queueP = lineage.findNode("nifi_queue", "queue", "4f3bfa4c-6427-3aac");
    final Node queueU = lineage.findNode("nifi_queue", "queue", "bb530e58-ee14-3cac");
    final Node pathL = lineage.findNode("nifi_flow_path", "LogAttribute", "97cc5b27-22f3-3c3b");
    final Node pathP = lineage.findNode("nifi_flow_path", "PutFile", "4f3bfa4c-6427-3aac");
    final Node pathU = lineage.findNode("nifi_flow_path", "UpdateAttribute", "bb530e58-ee14-3cac");
    // Remote flow owns the outputPort and transfer data generated by GenerateFlowFile.
    lineage.assertLink(remoteFlow, remoteGen);
    lineage.assertLink(remoteGen, outputPort);
    // The Remote Output Port path in local flow gets data from the remote.
    lineage.assertLink(localFlow, remoteOutputPortP);
    lineage.assertLink(outputPort, remoteOutputPortP);
    lineage.assertLink(remoteOutputPortP, queueL);
    lineage.assertLink(remoteOutputPortP, queueP);
    lineage.assertLink(remoteOutputPortP, queueU);
    lineage.assertLink(queueL, pathL);
    lineage.assertLink(queueP, pathP);
    lineage.assertLink(queueU, pathU);
}