use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForExpandChildren.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForExpandChildren() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
final ProvenanceEventRecord firstEvent = createEvent("4444");
final Map<String, String> previousAttributes = new HashMap<>();
previousAttributes.put("uuid", "4444");
final Map<String, String> updatedAttributes = new HashMap<>();
updatedAttributes.put("updated", "true");
final ProvenanceEventRecord fork = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.FORK).setAttributes(previousAttributes, updatedAttributes).addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("4444").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
index.addEvents(eventStore.addEvent(fork).getStorageLocations());
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final NiFiUser user = createUser();
final EventAuthorizer allowForkEvents = new EventAuthorizer() {
@Override
public boolean isAuthorized(ProvenanceEventRecord event) {
return event.getEventType() == ProvenanceEventType.FORK;
}
@Override
public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
}
};
List<LineageNode> nodes = Collections.emptyList();
while (nodes.size() < 5) {
final ComputeLineageSubmission submission = index.submitExpandChildren(1L, user, allowForkEvents);
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
nodes = submission.getResult().getNodes();
Thread.sleep(25L);
}
assertEquals(5, nodes.size());
assertEquals(1L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.FLOWFILE_NODE).count());
assertEquals(4L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).count());
final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
assertEquals(2, eventMap.size());
assertEquals(1, eventMap.get(ProvenanceEventType.FORK).size());
assertEquals(3, eventMap.get(ProvenanceEventType.UNKNOWN).size());
}
use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.
the class ITReportLineageToAtlas method compositeLineages.
private ComputeLineageResult compositeLineages(ComputeLineageResult... results) throws InterruptedException {
final ComputeLineageResult lineage = mock(ComputeLineageResult.class);
when(lineage.awaitCompletion(anyLong(), any())).thenReturn(true);
final List<LineageEdge> edges = new ArrayList<>();
final Set<LineageNode> nodes = new LinkedHashSet<>();
for (int i = 0; i < results.length; i++) {
edges.addAll(results[i].getEdges());
nodes.addAll(results[i].getNodes());
}
when(lineage.getEdges()).thenReturn(edges);
when(lineage.getNodes()).thenReturn(new ArrayList<>(nodes));
return lineage;
}
use of org.apache.nifi.provenance.lineage.LineageNode in project nifi by apache.
the class DtoFactory method createLineageDto.
/**
* Creates a LineageDTO for the specified Lineage.
*
* @param computeLineageSubmission submission
* @return dto
*/
public LineageDTO createLineageDto(final ComputeLineageSubmission computeLineageSubmission) {
// build the lineage dto
final LineageDTO dto = new LineageDTO();
final LineageRequestDTO requestDto = new LineageRequestDTO();
final LineageResultsDTO resultsDto = new LineageResultsDTO();
// include the original request and results
dto.setRequest(requestDto);
dto.setResults(resultsDto);
// rebuild the request from the submission object
switch(computeLineageSubmission.getLineageComputationType()) {
case EXPAND_CHILDREN:
requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
requestDto.setLineageRequestType(LineageRequestType.CHILDREN);
break;
case EXPAND_PARENTS:
requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
requestDto.setLineageRequestType(LineageRequestType.PARENTS);
break;
case FLOWFILE_LINEAGE:
final Collection<String> uuids = computeLineageSubmission.getLineageFlowFileUuids();
if (uuids.size() == 1) {
requestDto.setUuid(uuids.iterator().next());
}
requestDto.setEventId(computeLineageSubmission.getExpandedEventId());
requestDto.setLineageRequestType(LineageRequestType.FLOWFILE);
break;
}
// include lineage details
dto.setId(computeLineageSubmission.getLineageIdentifier());
dto.setSubmissionTime(computeLineageSubmission.getSubmissionTime());
// create the results dto
final ComputeLineageResult results = computeLineageSubmission.getResult();
dto.setFinished(results.isFinished());
dto.setPercentCompleted(results.getPercentComplete());
dto.setExpiration(results.getExpiration());
final List<LineageNode> nodes = results.getNodes();
final List<LineageEdge> edges = results.getEdges();
final List<ProvenanceNodeDTO> nodeDtos = new ArrayList<>();
if (results.isFinished()) {
// create the node dto's
for (final LineageNode node : nodes) {
switch(node.getNodeType()) {
case FLOWFILE_NODE:
nodeDtos.add(createFlowFileNodeDTO(node));
break;
case PROVENANCE_EVENT_NODE:
nodeDtos.add(createProvenanceEventNodeDTO((ProvenanceEventLineageNode) node));
break;
}
}
}
resultsDto.setNodes(nodeDtos);
// include any errors
if (results.getError() != null) {
final Set<String> errors = new HashSet<>();
errors.add(results.getError());
resultsDto.setErrors(errors);
}
// create the link dto's
final List<ProvenanceLinkDTO> linkDtos = new ArrayList<>();
for (final LineageEdge edge : edges) {
linkDtos.add(createProvenanceLinkDTO(edge));
}
resultsDto.setLinks(linkDtos);
return dto;
}
Aggregations