Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 31 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestPutElasticsearchHttpRecord method testPutElasticSearchOnTriggerQueryParameter.

@Test
public void testPutElasticSearchOnTriggerQueryParameter() throws IOException {
    // no failures
    PutElasticsearchHttpRecordTestProcessor p = new PutElasticsearchHttpRecordTestProcessor(false);
    p.setExpectedUrl("http://127.0.0.1:9200/_bulk?pipeline=my-pipeline");
    runner = TestRunners.newTestRunner(p);
    generateTestData();
    runner.setValidateExpressionUsage(true);
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL, "http://127.0.0.1:9200");
    runner.setProperty(PutElasticsearchHttpRecord.INDEX, "doc");
    runner.setProperty(PutElasticsearchHttpRecord.TYPE, "status");
    runner.setProperty(PutElasticsearchHttpRecord.ID_RECORD_PATH, "/id");
    // Set dynamic property, to be added to the URL as a query parameter
    runner.setProperty("pipeline", "my-pipeline");
    runner.enqueue(new byte[0], new HashMap<String, String>() {

        {
            put("doc_id", "28039652140");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(PutElasticsearchHttpRecord.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(PutElasticsearchHttpRecord.REL_SUCCESS).get(0);
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "28039652140");
    List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertNotNull(provEvents);
    assertEquals(1, provEvents.size());
    assertEquals(ProvenanceEventType.SEND, provEvents.get(0).getEventType());
}

Example 32 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class CompleteFlowPathLineage method extractLineagePaths.

private void extractLineagePaths(AnalysisContext context, Map<String, List<LineageNode>> lineageTree, LineagePath lineagePath, ProvenanceEventRecord lastEvent) {
    lineagePath.getEvents().add(lastEvent);
    List<LineageNode> parentEvents = findParentEvents(lineageTree, lastEvent);
    final boolean createSeparateParentPath = lineagePath.shouldCreateSeparatePath(lastEvent.getEventType());
    if (createSeparateParentPath && (parentEvents == null || parentEvents.isEmpty())) {
        // Try expanding the lineage.
        // This is for the FlowFiles those are FORKed (or JOINed ... etc) other FlowFile(s).
        // FlowFiles routed to 'original' may have these event types, too, however they have parents fetched together.
        // For example, with these inputs: CREATE(F1), FORK (F1 -> F2, F3), DROP(F1), SEND (F2), SEND(F3), DROP(F2), DROP(F3)
        // Then when DROP(F1) is queried, FORK(F1) and CREATE(F1) are returned.
        // For DROP(F2), SEND(F2) and FORK(F2) are returned.
        // For DROP(F3), SEND(F3) and FORK(F3) are returned.
        // In this case, FORK(F2) and FORK(F3) have to query their parents again, to get CREATE(F1).
        final ComputeLineageResult joinedParents = context.findParents(lastEvent.getEventId());
        analyzeLineageTree(joinedParents, lineageTree);
        parentEvents = findParentEvents(lineageTree, lastEvent);
    }
    if (parentEvents == null || parentEvents.isEmpty()) {
        logger.debug("{} does not have any parent, stop extracting lineage path.", lastEvent);
        return;
    }
    if (createSeparateParentPath) {
        // Treat those as separated lineage_path
        parentEvents.stream().map(parentEvent -> context.getProvenanceEvent(Long.parseLong(parentEvent.getIdentifier()))).filter(Objects::nonNull).forEach(parent -> {
            final LineagePath parentPath = new LineagePath();
            lineagePath.getParents().add(parentPath);
            extractLineagePaths(context, lineageTree, parentPath, parent);
        });
    } else {
        // Simply traverse upwards.
        if (parentEvents.size() > 1) {
            throw new IllegalStateException(String.format("Having more than 1 parents for event type %s" + " is not expected. Should ask NiFi developer for investigation. %s", lastEvent.getEventType(), lastEvent));
        }
        final ProvenanceEventRecord parentEvent = context.getProvenanceEvent(Long.parseLong(parentEvents.get(0).getIdentifier()));
        if (parentEvent != null) {
            extractLineagePaths(context, lineageTree, lineagePath, parentEvent);
        }
    }
}

Example 33 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class CompleteFlowPathLineage method analyzeLineagePath.

private void analyzeLineagePath(AnalysisContext analysisContext, LineagePath lineagePath) {
    final List<ProvenanceEventRecord> events = lineagePath.getEvents();
    final DataSetRefs parentRefs = new DataSetRefs(events.get(0).getComponentId());
    events.forEach(event -> {
        final DataSetRefs refs = executeAnalyzer(analysisContext, event);
        if (refs == null || refs.isEmpty()) {
            return;
        }
        refs.getInputs().forEach(parentRefs::addInput);
        refs.getOutputs().forEach(parentRefs::addOutput);
    });
    lineagePath.setRefs(parentRefs);
    // Analyse parents.
    lineagePath.getParents().forEach(parent -> analyzeLineagePath(analysisContext, parent));
}

Example 34 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class ReportLineageToAtlas method consumeNiFiProvenanceEvents.

private void consumeNiFiProvenanceEvents(ReportingContext context, NiFiFlow nifiFlow) {
    final EventAccess eventAccess = context.getEventAccess();
    final AnalysisContext analysisContext = new StandardAnalysisContext(nifiFlow, clusterResolvers, // FIXME: This class cast shouldn't be necessary to query lineage. Possible refactor target in next major update.
    (ProvenanceRepository) eventAccess.getProvenanceRepository());
    consumer.consumeEvents(context, (componentMapHolder, events) -> {
        for (ProvenanceEventRecord event : events) {
            try {
                lineageStrategy.processEvent(analysisContext, nifiFlow, event);
            } catch (Exception e) {
                // If something went wrong, log it and continue with other records.
                getLogger().error("Skipping failed analyzing event {} due to {}.", new Object[] { event, e, e });
            }
        }
        nifiAtlasHook.commitMessages();
    });
}

Example 35 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestExtractMediaMetadata method testProvenance.

@Test
public void testProvenance() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ExtractMediaMetadata());
    runner.setProperty(ExtractMediaMetadata.METADATA_KEY_FILTER, "");
    runner.setProperty(ExtractMediaMetadata.METADATA_KEY_PREFIX, "txt.");
    runner.assertValid();
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "test1.txt");
    runner.enqueue("test1".getBytes(), attrs);
    runner.run();
    runner.assertAllFlowFilesTransferred(ExtractMediaMetadata.SUCCESS, 1);
    runner.assertTransferCount(ExtractMediaMetadata.FAILURE, 0);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals(ExtractMediaMetadata.class.getSimpleName(), event.getComponentType());
    assertEquals("media attributes extracted", event.getDetails());
    assertEquals(ProvenanceEventType.ATTRIBUTES_MODIFIED, event.getEventType());
}