Examples with DataSetRefs org.apache.nifi.atlas.provenance.DataSetRefs used on opensource projects

Example 11 with DataSetRefs

use of org.apache.nifi.atlas.provenance.DataSetRefs in project nifi by apache.

the class CompleteFlowPathLineage method analyzeLineagePath.

private void analyzeLineagePath(AnalysisContext analysisContext, LineagePath lineagePath) {
    final List<ProvenanceEventRecord> events = lineagePath.getEvents();
    final DataSetRefs parentRefs = new DataSetRefs(events.get(0).getComponentId());
    events.forEach(event -> {
        final DataSetRefs refs = executeAnalyzer(analysisContext, event);
        if (refs == null || refs.isEmpty()) {
            return;
        }
        refs.getInputs().forEach(parentRefs::addInput);
        refs.getOutputs().forEach(parentRefs::addOutput);
    });
    lineagePath.setRefs(parentRefs);
    // Analyse parents.
    lineagePath.getParents().forEach(parent -> analyzeLineagePath(analysisContext, parent));
}

Example 12 with DataSetRefs

use of org.apache.nifi.atlas.provenance.DataSetRefs in project nifi by apache.

the class PutHiveStreaming method analyze.

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    final URI uri = parseUri(event.getTransitUri());
    final String clusterName = context.getClusterResolver().fromHostNames(uri.getHost());
    final Set<Tuple<String, String>> outputTables = parseTableNames(null, event.getAttribute(ATTR_OUTPUT_TABLES));
    if (outputTables.isEmpty()) {
        return null;
    }
    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    outputTables.forEach(tableName -> {
        final Referenceable ref = createTableRef(clusterName, tableName);
        refs.addOutput(ref);
    });
    return refs;
}

Example 13 with DataSetRefs

use of org.apache.nifi.atlas.provenance.DataSetRefs in project nifi by apache.

the class UnknownInput method analyze.

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addInput(ref);
    return refs;
}

Example 14 with DataSetRefs

use of org.apache.nifi.atlas.provenance.DataSetRefs in project nifi by apache.

the class UnknownOutput method analyze.

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addOutput(ref);
    return refs;
}

Example 15 with DataSetRefs

use of org.apache.nifi.atlas.provenance.DataSetRefs in project nifi by apache.

the class CompleteFlowPathLineage method processEvent.

@Override
public void processEvent(AnalysisContext analysisContext, NiFiFlow nifiFlow, ProvenanceEventRecord event) {
    if (!ProvenanceEventType.DROP.equals(event.getEventType())) {
        return;
    }
    final ComputeLineageResult lineage = analysisContext.queryLineage(event.getEventId());
    // Construct a tree model to traverse backwards.
    final Map<String, List<LineageNode>> lineageTree = new HashMap<>();
    analyzeLineageTree(lineage, lineageTree);
    final LineagePath lineagePath = new LineagePath();
    extractLineagePaths(analysisContext, lineageTree, lineagePath, event);
    analyzeLineagePath(analysisContext, lineagePath);
    // Input and output data set are both required to report lineage.
    List<Tuple<NiFiFlowPath, DataSetRefs>> createdFlowPaths = new ArrayList<>();
    if (lineagePath.isComplete()) {
        createCompleteFlowPath(nifiFlow, lineagePath, createdFlowPaths);
        for (Tuple<NiFiFlowPath, DataSetRefs> createdFlowPath : createdFlowPaths) {
            final NiFiFlowPath flowPath = createdFlowPath.getKey();
            createEntity(toReferenceable(flowPath, nifiFlow));
            addDataSetRefs(nifiFlow, Collections.singleton(flowPath), createdFlowPath.getValue());
        }
        createdFlowPaths.clear();
    }
}