Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 16 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestStandardProcessSession method testCreateEmitted.

@Test
public void testCreateEmitted() throws IOException {
    final FlowFile newFlowFile = session.create();
    session.transfer(newFlowFile, new Relationship.Builder().name("A").build());
    session.commit();
    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 10000);
    assertFalse(events.isEmpty());
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals(ProvenanceEventType.CREATE, event.getEventType());
}

Example 17 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestConsumeAzureEventHub method testCheckpointFailure.

@Test
public void testCheckpointFailure() throws Exception {
    final Iterable<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)));
    doThrow(new RuntimeException("Failed to create a checkpoint.")).when(partitionContext).checkpoint();
    eventProcessor.onEvents(partitionContext, eventDataList);
    // Even if it fails to create a checkpoint, these FlowFiles are already committed.
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(2, flowFiles.size());
    final MockFlowFile msg1 = flowFiles.get(0);
    msg1.assertContentEquals("one");
    final MockFlowFile msg2 = flowFiles.get(1);
    msg2.assertContentEquals("two");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
}

Example 18 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestConsumeAzureEventHub method testReceiveRecordReaderFailure.

@Test
public void testReceiveRecordReaderFailure() throws Exception {
    final List<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)), new EventData("three".getBytes(StandardCharsets.UTF_8)), new EventData("four".getBytes(StandardCharsets.UTF_8)));
    setupRecordReader(eventDataList, 2, null);
    setupRecordWriter();
    eventProcessor.onEvents(partitionContext, eventDataList);
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(1, flowFiles.size());
    final MockFlowFile ff1 = flowFiles.get(0);
    ff1.assertContentEquals("onetwofour");
    ff1.assertAttributeEquals("eventhub.name", "eventhub-name");
    ff1.assertAttributeEquals("eventhub.partition", "partition-id");
    final List<MockFlowFile> failedFFs = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_PARSE_FAILURE);
    assertEquals(1, failedFFs.size());
    final MockFlowFile failed1 = failedFFs.get(0);
    failed1.assertContentEquals("three");
    failed1.assertAttributeEquals("eventhub.name", "eventhub-name");
    failed1.assertAttributeEquals("eventhub.partition", "partition-id");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent1 = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent1.getEventType());
    assertEquals("amqps://namespace.servicebus.windows.net/" + "eventhub-name/ConsumerGroups/consumer-group/Partitions/partition-id", provenanceEvent1.getTransitUri());
    final ProvenanceEventRecord provenanceEvent2 = provenanceEvents.get(1);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent2.getEventType());
    assertEquals("amqps://namespace.servicebus.windows.net/" + "eventhub-name/ConsumerGroups/consumer-group/Partitions/partition-id", provenanceEvent2.getTransitUri());
}

Example 19 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestConsumeAzureEventHub method testReceiveTwo.

@Test
public void testReceiveTwo() throws Exception {
    final Iterable<EventData> eventDataList = Arrays.asList(new EventData("one".getBytes(StandardCharsets.UTF_8)), new EventData("two".getBytes(StandardCharsets.UTF_8)));
    eventProcessor.onEvents(partitionContext, eventDataList);
    processSession.assertCommitted();
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(ConsumeAzureEventHub.REL_SUCCESS);
    assertEquals(2, flowFiles.size());
    final MockFlowFile msg1 = flowFiles.get(0);
    msg1.assertContentEquals("one");
    final MockFlowFile msg2 = flowFiles.get(1);
    msg2.assertContentEquals("two");
    final List<ProvenanceEventRecord> provenanceEvents = sharedState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
}

Example 20 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestHBaseTable method testHBaseTableWithMultipleZkHosts.

@Test
public void testHBaseTableWithMultipleZkHosts() {
    final String processorName = "FetchHBaseRow";
    final String transitUri = "hbase://zk0.example.com,zk2.example.com,zk3.example.com/tableA/rowB";
    final ProvenanceEventRecord record = Mockito.mock(ProvenanceEventRecord.class);
    when(record.getComponentType()).thenReturn(processorName);
    when(record.getTransitUri()).thenReturn(transitUri);
    when(record.getEventType()).thenReturn(ProvenanceEventType.FETCH);
    final ClusterResolvers clusterResolvers = Mockito.mock(ClusterResolvers.class);
    when(clusterResolvers.fromHostNames(matches("zk0.example.com"), matches("zk2.example.com"), matches("zk3.example.com"))).thenReturn("cluster1");
    final AnalysisContext context = Mockito.mock(AnalysisContext.class);
    when(context.getClusterResolver()).thenReturn(clusterResolvers);
    final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(processorName, transitUri, record.getEventType());
    assertNotNull(analyzer);
    final DataSetRefs refs = analyzer.analyze(context, record);
    assertEquals(1, refs.getInputs().size());
    assertEquals(0, refs.getOutputs().size());
    Referenceable ref = refs.getInputs().iterator().next();
    assertEquals("hbase_table", ref.getTypeName());
    assertEquals("tableA", ref.get(ATTR_NAME));
    assertEquals("tableA@cluster1", ref.get(ATTR_QUALIFIED_NAME));
}