Examples with LineageEdge org.apache.nifi.provenance.lineage.LineageEdge used on opensource projects

Example 1 with LineageEdge

use of org.apache.nifi.provenance.lineage.LineageEdge in project nifi by apache.

the class ITReportLineageToAtlas method createLineage.

private ComputeLineageResult createLineage(ProvenanceRecords prs, int... indices) throws InterruptedException {
    final ComputeLineageResult lineage = mock(ComputeLineageResult.class);
    when(lineage.awaitCompletion(anyLong(), any())).thenReturn(true);
    final List<LineageEdge> edges = new ArrayList<>();
    final Set<LineageNode> nodes = new LinkedHashSet<>();
    for (int i = 0; i < indices.length - 1; i++) {
        final EdgeNode edge = createEdge(prs, indices[i], indices[i + 1]);
        edges.add(edge);
        nodes.add(edge.getSource());
        nodes.add(edge.getDestination());
    }
    when(lineage.getEdges()).thenReturn(edges);
    when(lineage.getNodes()).thenReturn(new ArrayList<>(nodes));
    return lineage;
}

Example 2 with LineageEdge

use of org.apache.nifi.provenance.lineage.LineageEdge in project nifi by apache.

the class TestPersistentProvenanceRepository method testLineageManyToOneSpawn.

@Test
public void testLineageManyToOneSpawn() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(3, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String childId = "00000000-0000-0000-0000-000000000000";
    final String parentId1 = "00000000-0000-0000-0001-000000000001";
    final String parentId2 = "00000000-0000-0000-0001-000000000002";
    final String parentId3 = "00000000-0000-0000-0001-000000000003";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("uuid", childId);
    attributes.put("filename", "file-" + childId);
    final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.FORK);
    attributes.put("uuid", childId);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.addChildUuid(childId);
    builder.addParentUuid(parentId1);
    builder.addParentUuid(parentId2);
    builder.addParentUuid(parentId3);
    repo.registerEvent(builder.build());
    repo.waitForRollover();
    final Lineage lineage = repo.computeLineage(childId, createUser());
    assertNotNull(lineage);
    // these are not necessarily accurate asserts....
    final List<LineageNode> nodes = lineage.getNodes();
    final List<LineageEdge> edges = lineage.getEdges();
    assertEquals(2, nodes.size());
    assertEquals(1, edges.size());
}

Example 3 with LineageEdge

use of org.apache.nifi.provenance.lineage.LineageEdge in project nifi by apache.

the class TestPersistentProvenanceRepository method testLineageReceiveDrop.

@Test
public void testLineageReceiveDrop() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(3, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000001";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("uuid", uuid);
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    repo.registerEvent(builder.build());
    builder.setEventTime(System.currentTimeMillis() + 1);
    builder.setEventType(ProvenanceEventType.DROP);
    builder.setTransitUri(null);
    repo.registerEvent(builder.build());
    repo.waitForRollover();
    final Lineage lineage = repo.computeLineage(uuid, createUser());
    assertNotNull(lineage);
    // Nodes should consist of a RECEIVE followed by FlowFileNode, followed by a DROP
    final List<LineageNode> nodes = lineage.getNodes();
    final List<LineageEdge> edges = lineage.getEdges();
    assertEquals(3, nodes.size());
    for (final LineageEdge edge : edges) {
        if (edge.getSource().getNodeType() == LineageNodeType.FLOWFILE_NODE) {
            assertTrue(edge.getDestination().getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE);
            assertTrue(((EventNode) edge.getDestination()).getEventType() == ProvenanceEventType.DROP);
        } else {
            assertTrue(((EventNode) edge.getSource()).getEventType() == ProvenanceEventType.RECEIVE);
            assertTrue(edge.getDestination().getNodeType() == LineageNodeType.FLOWFILE_NODE);
        }
    }
}

Example 4 with LineageEdge

use of org.apache.nifi.provenance.lineage.LineageEdge in project nifi by apache.

the class TestPersistentProvenanceRepository method testLineageReceiveDropAsync.

@Test
public void testLineageReceiveDropAsync() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(3, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String uuid = "00000000-0000-0000-0000-000000000001";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("uuid", uuid);
    attributes.put("filename", "file-" + uuid);
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", uuid);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    repo.registerEvent(builder.build());
    builder.setEventTime(System.currentTimeMillis() + 1);
    builder.setEventType(ProvenanceEventType.DROP);
    builder.setTransitUri(null);
    repo.registerEvent(builder.build());
    repo.waitForRollover();
    final AsyncLineageSubmission submission = repo.submitLineageComputation(uuid, createUser());
    while (!submission.getResult().isFinished()) {
        Thread.sleep(100L);
    }
    assertNotNull(submission);
    // Nodes should consist of a RECEIVE followed by FlowFileNode, followed by a DROP
    final List<LineageNode> nodes = submission.getResult().getNodes();
    final List<LineageEdge> edges = submission.getResult().getEdges();
    assertEquals(3, nodes.size());
    for (final LineageEdge edge : edges) {
        if (edge.getSource().getNodeType() == LineageNodeType.FLOWFILE_NODE) {
            assertTrue(edge.getDestination().getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE);
            assertTrue(((EventNode) edge.getDestination()).getEventType() == ProvenanceEventType.DROP);
        } else {
            assertTrue(((EventNode) edge.getSource()).getEventType() == ProvenanceEventType.RECEIVE);
            assertTrue(edge.getDestination().getNodeType() == LineageNodeType.FLOWFILE_NODE);
        }
    }
}

Example 5 with LineageEdge

use of org.apache.nifi.provenance.lineage.LineageEdge in project nifi by apache.

the class ITReportLineageToAtlas method compositeLineages.

private ComputeLineageResult compositeLineages(ComputeLineageResult... results) throws InterruptedException {
    final ComputeLineageResult lineage = mock(ComputeLineageResult.class);
    when(lineage.awaitCompletion(anyLong(), any())).thenReturn(true);
    final List<LineageEdge> edges = new ArrayList<>();
    final Set<LineageNode> nodes = new LinkedHashSet<>();
    for (int i = 0; i < results.length; i++) {
        edges.addAll(results[i].getEdges());
        nodes.addAll(results[i].getNodes());
    }
    when(lineage.getEdges()).thenReturn(edges);
    when(lineage.getNodes()).thenReturn(new ArrayList<>(nodes));
    return lineage;
}