Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 76 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestInvokeHttpCommon method test200.

@Test
public void test200() throws Exception {
    addHandler(new GetOrHeadHandler());
    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
    createFlowFiles(runner);
    // Verify only one FlowFile gets created/sent
    runner.run();
    runner.run();
    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);
    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");
    // expected in response
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
    bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "text/plain;charset=iso-8859-1");
    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(2, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        }
    }
    assertTrue(forkEvent);
    assertTrue(fetchEvent);
}

Example 77 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestListenSyslog method testBatching.

@Test
public void testBatching() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    runner.setProperty(ListenSyslog.MAX_BATCH_SIZE, "25");
    runner.setProperty(ListenSyslog.MESSAGE_DELIMITER, "|");
    runner.setProperty(ListenSyslog.PARSE_MESSAGES, "false");
    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);
    // the processor has internal blocking queue with capacity 10 so we have to send
    // less than that since we are sending all messages before the processors ever runs
    final int numMessages = 5;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);
    // write some UDP messages to the port in the background
    final Thread sender = new Thread(new DatagramSender(port, numMessages, 10, VALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();
    sender.join();
    try {
        proc.onTrigger(context, processSessionFactory);
        runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);
        final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        Assert.assertEquals("0", flowFile.getAttribute(SyslogAttributes.PORT.key()));
        Assert.assertEquals(ListenSyslog.UDP_VALUE.getValue(), flowFile.getAttribute(SyslogAttributes.PROTOCOL.key()));
        Assert.assertTrue(!StringUtils.isBlank(flowFile.getAttribute(SyslogAttributes.SENDER.key())));
        final String content = new String(flowFile.toByteArray(), StandardCharsets.UTF_8);
        final String[] splits = content.split("\\|");
        Assert.assertEquals(numMessages, splits.length);
        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(1, events.size());
        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("udp"));
    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 78 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestListenUDP method verifyProvenance.

private void verifyProvenance(int expectedNumEvents) {
    List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    Assert.assertEquals(expectedNumEvents, provEvents.size());
    for (ProvenanceEventRecord event : provEvents) {
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue(event.getTransitUri().startsWith("udp://"));
    }
}

Example 79 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestPutJMS method testPutProvenanceSendEventTransitUri.

@Test
public void testPutProvenanceSendEventTransitUri() throws JMSException {
    final PutJMS putJMS = spy(new PutJMS());
    final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
    runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
    runnerPut.setProperty(JmsProperties.URL, TEST_URL);
    runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
    runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
    runnerPut.setProperty(JmsProperties.ATTRIBUTES_TO_JMS_PROPS, "true");
    runnerPut.enqueue("putGetMessage".getBytes());
    runnerPut.run();
    assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
    assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());
    final List<ProvenanceEventRecord> putProvenanceEvents = runnerPut.getProvenanceEvents();
    assertEquals(1, putProvenanceEvents.size());
    // Verify the transitUri is the same as that configured in the properties
    assertEquals(TEST_URL, putProvenanceEvents.get(0).getTransitUri());
    final GetJMSQueue getJmsQueue = new GetJMSQueue();
    final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
    runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
    runnerGet.setProperty(JmsProperties.URL, TEST_URL);
    runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
    runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
    runnerGet.run();
    assertEquals(1, runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS).size());
    final List<MockFlowFile> flowFilesGet = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
    assertEquals(1, flowFilesGet.size());
    final MockFlowFile successFlowFile = flowFilesGet.get(0);
    successFlowFile.assertContentEquals("putGetMessage");
}

Example 80 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class MockProvenanceReporter method modifyAttributes.

@Override
public void modifyAttributes(final FlowFile flowFile, final String details) {
    verifyFlowFileKnown(flowFile);
    try {
        final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.ATTRIBUTES_MODIFIED).setDetails(details).build();
        events.add(record);
    } catch (final Exception e) {
        logger.error("Failed to generate Provenance Event due to " + e);
        if (logger.isDebugEnabled()) {
            logger.error("", e);
        }
    }
}