Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 66 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class ITListenSyslog method testUDP.

@Test
public void testUDP() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);
    final int numMessages = 20;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);
    // write some UDP messages to the port in the background
    final Thread sender = new Thread(new DatagramSender(port, numMessages, 10, VALID_MESSAGE));
    sender.setDaemon(true);
    sender.start();
    // call onTrigger until we read all datagrams, or 30 seconds passed
    try {
        int numTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;
        while (numTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            numTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the datagrams", numMessages, numTransferred);
        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.UDP_VALUE.getValue());
        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());
        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("udp"));
    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 67 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class ITListenSyslog method testTCPMultipleConnection.

@Test
public void testTCPMultipleConnection() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.MAX_CONNECTIONS, "5");
    runner.setProperty(ListenSyslog.PORT, "0");
    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);
    final int numMessages = 20;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);
    // write some TCP messages to the port in the background
    final Thread sender = new Thread(new MultiConnectionSocketSender(port, numMessages, 10, VALID_MESSAGE_TCP));
    sender.setDaemon(true);
    sender.start();
    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;
        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);
        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());
        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());
        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));
    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 68 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class ITListenSyslog method testTCPSingleConnectionWithNewLines.

@Test
public void testTCPSingleConnectionWithNewLines() throws IOException, InterruptedException {
    final ListenSyslog proc = new ListenSyslog();
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.TCP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    // schedule to start listening on a random port
    final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory();
    final ProcessContext context = runner.getProcessContext();
    proc.onScheduled(context);
    final int numMessages = 3;
    final int port = proc.getPort();
    Assert.assertTrue(port > 0);
    // send 3 messages as 1
    final String multipleMessages = VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n" + VALID_MESSAGE_TCP + "\n";
    final Thread sender = new Thread(new SingleConnectionSocketSender(port, 1, 10, multipleMessages));
    sender.setDaemon(true);
    sender.start();
    // call onTrigger until we read all messages, or 30 seconds passed
    try {
        int nubTransferred = 0;
        long timeout = System.currentTimeMillis() + 30000;
        while (nubTransferred < numMessages && System.currentTimeMillis() < timeout) {
            Thread.sleep(10);
            proc.onTrigger(context, processSessionFactory);
            nubTransferred = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).size();
        }
        Assert.assertEquals("Did not process all the messages", numMessages, nubTransferred);
        MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0);
        checkFlowFile(flowFile, 0, ListenSyslog.TCP_VALUE.getValue());
        final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
        Assert.assertNotNull(events);
        Assert.assertEquals(numMessages, events.size());
        final ProvenanceEventRecord event = events.get(0);
        Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
        Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));
    } finally {
        // unschedule to close connections
        proc.onUnscheduled();
    }
}

Example 69 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestGetSplunk method testGetWithProvidedTime.

@Test
public void testGetWithProvidedTime() {
    final String query = "search tcp:7879";
    final String providedEarliest = "-1h";
    final String providedLatest = "now";
    final String outputMode = GetSplunk.ATOM_VALUE.getValue();
    runner.setProperty(GetSplunk.QUERY, query);
    runner.setProperty(GetSplunk.EARLIEST_TIME, providedEarliest);
    runner.setProperty(GetSplunk.LATEST_TIME, providedLatest);
    runner.setProperty(GetSplunk.OUTPUT_MODE, outputMode);
    final JobExportArgs expectedArgs = new JobExportArgs();
    expectedArgs.setSearchMode(JobExportArgs.SearchMode.NORMAL);
    expectedArgs.setEarliestTime(providedEarliest);
    expectedArgs.setLatestTime(providedLatest);
    expectedArgs.setOutputMode(JobExportArgs.OutputMode.valueOf(outputMode));
    final String resultContent = "fake results";
    final ByteArrayInputStream input = new ByteArrayInputStream(resultContent.getBytes(StandardCharsets.UTF_8));
    when(service.export(eq(query), argThat(new JobExportArgsMatcher(expectedArgs)))).thenReturn(input);
    runner.run();
    runner.assertAllFlowFilesTransferred(GetSplunk.REL_SUCCESS, 1);
    final List<MockFlowFile> mockFlowFiles = runner.getFlowFilesForRelationship(GetSplunk.REL_SUCCESS);
    Assert.assertEquals(1, mockFlowFiles.size());
    final MockFlowFile mockFlowFile = mockFlowFiles.get(0);
    mockFlowFile.assertContentEquals(resultContent);
    mockFlowFile.assertAttributeEquals(GetSplunk.QUERY_ATTR, query);
    mockFlowFile.assertAttributeEquals(GetSplunk.EARLIEST_TIME_ATTR, providedEarliest);
    mockFlowFile.assertAttributeEquals(GetSplunk.LATEST_TIME_ATTR, providedLatest);
    Assert.assertEquals(1, proc.count);
    final List<ProvenanceEventRecord> events = runner.getProvenanceEvents();
    Assert.assertEquals(1, events.size());
    Assert.assertEquals(ProvenanceEventType.RECEIVE, events.get(0).getEventType());
    Assert.assertEquals("https://localhost:8089", events.get(0).getTransitUri());
}

Example 70 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestConnectWebSocket method testSuccess.

@Test
public void testSuccess() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(ConnectWebSocket.class);
    final ConnectWebSocket processor = (ConnectWebSocket) runner.getProcessor();
    final SharedSessionState sharedSessionState = new SharedSessionState(processor, new AtomicLong(0));
    // Use this custom session factory implementation so that createdSessions can be read from test case,
    // because MockSessionFactory doesn't expose it.
    final Set<MockProcessSession> createdSessions = new HashSet<>();
    final ProcessSessionFactory sessionFactory = () -> {
        final MockProcessSession session = new MockProcessSession(sharedSessionState, processor);
        createdSessions.add(session);
        return session;
    };
    final WebSocketClientService service = mock(WebSocketClientService.class);
    final WebSocketSession webSocketSession = spy(AbstractWebSocketSession.class);
    when(webSocketSession.getSessionId()).thenReturn("ws-session-id");
    when(webSocketSession.getLocalAddress()).thenReturn(new InetSocketAddress("localhost", 12345));
    when(webSocketSession.getRemoteAddress()).thenReturn(new InetSocketAddress("example.com", 80));
    final String serviceId = "ws-service";
    final String endpointId = "client-1";
    final String textMessageFromServer = "message from server.";
    when(service.getIdentifier()).thenReturn(serviceId);
    when(service.getTargetUri()).thenReturn("ws://example.com/web-socket");
    doAnswer(invocation -> {
        processor.connected(webSocketSession);
        // Two times.
        processor.consume(webSocketSession, textMessageFromServer);
        processor.consume(webSocketSession, textMessageFromServer);
        // Three times.
        final byte[] binaryMessage = textMessageFromServer.getBytes();
        processor.consume(webSocketSession, binaryMessage, 0, binaryMessage.length);
        processor.consume(webSocketSession, binaryMessage, 0, binaryMessage.length);
        processor.consume(webSocketSession, binaryMessage, 0, binaryMessage.length);
        return null;
    }).when(service).connect(endpointId);
    runner.addControllerService(serviceId, service);
    runner.enableControllerService(service);
    runner.setProperty(ConnectWebSocket.PROP_WEBSOCKET_CLIENT_SERVICE, serviceId);
    runner.setProperty(ConnectWebSocket.PROP_WEBSOCKET_CLIENT_ID, endpointId);
    processor.onTrigger(runner.getProcessContext(), sessionFactory);
    final Map<Relationship, List<MockFlowFile>> transferredFlowFiles = getAllTransferredFlowFiles(createdSessions, processor);
    List<MockFlowFile> connectedFlowFiles = transferredFlowFiles.get(AbstractWebSocketGatewayProcessor.REL_CONNECTED);
    assertEquals(1, connectedFlowFiles.size());
    connectedFlowFiles.forEach(ff -> {
        assertFlowFile(webSocketSession, serviceId, endpointId, ff, null);
    });
    List<MockFlowFile> textFlowFiles = transferredFlowFiles.get(AbstractWebSocketGatewayProcessor.REL_MESSAGE_TEXT);
    assertEquals(2, textFlowFiles.size());
    textFlowFiles.forEach(ff -> {
        assertFlowFile(webSocketSession, serviceId, endpointId, ff, WebSocketMessage.Type.TEXT);
    });
    List<MockFlowFile> binaryFlowFiles = transferredFlowFiles.get(AbstractWebSocketGatewayProcessor.REL_MESSAGE_BINARY);
    assertEquals(3, binaryFlowFiles.size());
    binaryFlowFiles.forEach(ff -> {
        assertFlowFile(webSocketSession, serviceId, endpointId, ff, WebSocketMessage.Type.BINARY);
    });
    final List<ProvenanceEventRecord> provenanceEvents = sharedSessionState.getProvenanceEvents();
    assertEquals(6, provenanceEvents.size());
    assertTrue(provenanceEvents.stream().allMatch(event -> ProvenanceEventType.RECEIVE.equals(event.getEventType())));
}