Examples with LineageNode org.apache.nifi.provenance.lineage.LineageNode used on opensource projects

Example 1 with LineageNode

use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.

the class DtoFactory method createProvenanceLinkDTO.

/**
 * Creates a ProvenanceLinkDTO for the specified LineageEdge.
 *
 * @param edge edge
 * @return dto
 */
public ProvenanceLinkDTO createProvenanceLinkDTO(final LineageEdge edge) {
    final LineageNode source = edge.getSource();
    final LineageNode target = edge.getDestination();
    final ProvenanceLinkDTO dto = new ProvenanceLinkDTO();
    dto.setTimestamp(new Date(target.getTimestamp()));
    dto.setMillis(target.getTimestamp());
    dto.setFlowFileUuid(edge.getUuid());
    dto.setSourceId(source.getIdentifier());
    dto.setTargetId(target.getIdentifier());
    return dto;
}

Example 2 with LineageNode

use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.

the class ITReportLineageToAtlas method createLineage.

private ComputeLineageResult createLineage(ProvenanceRecords prs, int... indices) throws InterruptedException {
    final ComputeLineageResult lineage = mock(ComputeLineageResult.class);
    when(lineage.awaitCompletion(anyLong(), any())).thenReturn(true);
    final List<LineageEdge> edges = new ArrayList<>();
    final Set<LineageNode> nodes = new LinkedHashSet<>();
    for (int i = 0; i < indices.length - 1; i++) {
        final EdgeNode edge = createEdge(prs, indices[i], indices[i + 1]);
        edges.add(edge);
        nodes.add(edge.getSource());
        nodes.add(edge.getDestination());
    }
    when(lineage.getEdges()).thenReturn(edges);
    when(lineage.getNodes()).thenReturn(new ArrayList<>(nodes));
    return lineage;
}

Example 3 with LineageNode

use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.

the class CompleteFlowPathLineage method extractLineagePaths.

private void extractLineagePaths(AnalysisContext context, Map<String, List<LineageNode>> lineageTree, LineagePath lineagePath, ProvenanceEventRecord lastEvent) {
    lineagePath.getEvents().add(lastEvent);
    List<LineageNode> parentEvents = findParentEvents(lineageTree, lastEvent);
    final boolean createSeparateParentPath = lineagePath.shouldCreateSeparatePath(lastEvent.getEventType());
    if (createSeparateParentPath && (parentEvents == null || parentEvents.isEmpty())) {
        // Try expanding the lineage.
        // This is for the FlowFiles those are FORKed (or JOINed ... etc) other FlowFile(s).
        // FlowFiles routed to 'original' may have these event types, too, however they have parents fetched together.
        // For example, with these inputs: CREATE(F1), FORK (F1 -> F2, F3), DROP(F1), SEND (F2), SEND(F3), DROP(F2), DROP(F3)
        // Then when DROP(F1) is queried, FORK(F1) and CREATE(F1) are returned.
        // For DROP(F2), SEND(F2) and FORK(F2) are returned.
        // For DROP(F3), SEND(F3) and FORK(F3) are returned.
        // In this case, FORK(F2) and FORK(F3) have to query their parents again, to get CREATE(F1).
        final ComputeLineageResult joinedParents = context.findParents(lastEvent.getEventId());
        analyzeLineageTree(joinedParents, lineageTree);
        parentEvents = findParentEvents(lineageTree, lastEvent);
    }
    if (parentEvents == null || parentEvents.isEmpty()) {
        logger.debug("{} does not have any parent, stop extracting lineage path.", lastEvent);
        return;
    }
    if (createSeparateParentPath) {
        // Treat those as separated lineage_path
        parentEvents.stream().map(parentEvent -> context.getProvenanceEvent(Long.parseLong(parentEvent.getIdentifier()))).filter(Objects::nonNull).forEach(parent -> {
            final LineagePath parentPath = new LineagePath();
            lineagePath.getParents().add(parentPath);
            extractLineagePaths(context, lineageTree, parentPath, parent);
        });
    } else {
        // Simply traverse upwards.
        if (parentEvents.size() > 1) {
            throw new IllegalStateException(String.format("Having more than 1 parents for event type %s" + " is not expected. Should ask NiFi developer for investigation. %s", lastEvent.getEventType(), lastEvent));
        }
        final ProvenanceEventRecord parentEvent = context.getProvenanceEvent(Long.parseLong(parentEvents.get(0).getIdentifier()));
        if (parentEvent != null) {
            extractLineagePaths(context, lineageTree, lineagePath, parentEvent);
        }
    }
}

Example 4 with LineageNode

use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.

the class SimpleFlowPathLineage method findPreviousProvenanceEvent.

private ProvenanceEventRecord findPreviousProvenanceEvent(AnalysisContext context, ProvenanceEventRecord event) {
    final ComputeLineageResult lineage = context.queryLineage(event.getEventId());
    if (lineage == null) {
        logger.warn("Lineage was not found: {}", new Object[] { event });
        return null;
    }
    // If no previous provenance node found due to expired or other reasons, just log a warning msg and do nothing.
    final LineageNode previousProvenanceNode = traverseLineage(lineage, String.valueOf(event.getEventId()));
    if (previousProvenanceNode == null) {
        logger.warn("Traverse lineage could not find any preceding provenance event node: {}", new Object[] { event });
        return null;
    }
    final long previousEventId = Long.parseLong(previousProvenanceNode.getIdentifier());
    return context.getProvenanceEvent(previousEventId);
}

Example 5 with LineageNode

use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.

the class TestPersistentProvenanceRepository method testLineageManyToOneSpawn.

@Test
public void testLineageManyToOneSpawn() throws IOException, InterruptedException, ParseException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxRecordLife(3, TimeUnit.SECONDS);
    config.setMaxStorageCapacity(1024L * 1024L);
    config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);
    final String childId = "00000000-0000-0000-0000-000000000000";
    final String parentId1 = "00000000-0000-0000-0001-000000000001";
    final String parentId2 = "00000000-0000-0000-0001-000000000002";
    final String parentId3 = "00000000-0000-0000-0001-000000000003";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("uuid", childId);
    attributes.put("filename", "file-" + childId);
    final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.FORK);
    attributes.put("uuid", childId);
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.addChildUuid(childId);
    builder.addParentUuid(parentId1);
    builder.addParentUuid(parentId2);
    builder.addParentUuid(parentId3);
    repo.registerEvent(builder.build());
    repo.waitForRollover();
    final Lineage lineage = repo.computeLineage(childId, createUser());
    assertNotNull(lineage);
    // these are not necessarily accurate asserts....
    final List<LineageNode> nodes = lineage.getNodes();
    final List<LineageEdge> edges = lineage.getEdges();
    assertEquals(2, nodes.size());
    assertEquals(1, edges.size());
}