Examples with ProvenanceEventRecord org.apache.nifi.provenance.ProvenanceEventRecord used on opensource projects

Example 11 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestHttpFlowFileServerProtocol method testTransferTwoFiles.

@Test
public void testTransferTwoFiles() throws Exception {
    final String transactionId = "testTransferTwoFiles";
    final Peer peer = getDefaultPeer(transactionId);
    final String endpointUri = "https://remote-host:8443/nifi-api/output-ports/port-id/transactions/" + transactionId + "/flow-files";
    final HttpFlowFileServerProtocol serverProtocol = getDefaultHttpFlowFileServerProtocol();
    final HttpServerCommunicationsSession commsSession = (HttpServerCommunicationsSession) peer.getCommunicationsSession();
    commsSession.putHandshakeParam(HandshakeProperty.BATCH_COUNT, "2");
    commsSession.setUserDn("unit-test");
    commsSession.setDataTransferUrl(endpointUri);
    transferFlowFiles(serverProtocol, transactionId, peer, processSession -> IntStream.of(1, 2).mapToObj(i -> {
        final MockFlowFile flowFile = processSession.createFlowFile(("Server content " + i).getBytes());
        final HashMap<String, String> attributes = new HashMap<>();
        attributes.put("uuid", "server-uuid-" + i);
        attributes.put("filename", "server-filename-" + i);
        attributes.put("server-attr-" + i + "-1", "server-attr-" + i + "-1-value");
        attributes.put("server-attr-" + i + "-2", "server-attr-" + i + "-2-value");
        flowFile.putAttributes(attributes);
        return flowFile;
    }).collect(Collectors.toList()));
    // Commit transaction
    final int flowFileSent = serverProtocol.commitTransferTransaction(peer, "3058746557");
    assertEquals(2, flowFileSent);
    // Assert provenance
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
    for (final ProvenanceEventRecord provenanceEvent : provenanceEvents) {
        assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
        assertEquals(endpointUri, provenanceEvent.getTransitUri());
        assertEquals("Remote Host=peer-host, Remote DN=unit-test", provenanceEvent.getDetails());
    }
}

Example 12 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestHttpFlowFileServerProtocol method testReceiveTwoFiles.

@Test
public void testReceiveTwoFiles() throws Exception {
    final String transactionId = "testReceiveTwoFile";
    final String endpointUri = "https://remote-host:8443/nifi-api/input-ports/port-id/transactions/" + transactionId + "/flow-files";
    final HttpFlowFileServerProtocol serverProtocol = getDefaultHttpFlowFileServerProtocol();
    final Peer peer = getDefaultPeer(transactionId);
    final HttpServerCommunicationsSession commsSession = (HttpServerCommunicationsSession) peer.getCommunicationsSession();
    commsSession.putHandshakeParam(HandshakeProperty.BATCH_COUNT, "2");
    commsSession.setUserDn("unit-test");
    commsSession.setDataTransferUrl(endpointUri);
    receiveFlowFiles(serverProtocol, transactionId, peer, createClientDataPacket(), createClientDataPacket());
    // Commit transaction
    commsSession.setResponseCode(ResponseCode.CONFIRM_TRANSACTION);
    final int flowFileReceived = serverProtocol.commitReceiveTransaction(peer);
    assertEquals(2, flowFileReceived);
    // Assert provenance.
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(2, provenanceEvents.size());
    for (final ProvenanceEventRecord provenanceEvent : provenanceEvents) {
        assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
        assertEquals(endpointUri, provenanceEvent.getTransitUri());
        assertEquals("Remote Host=peer-host, Remote DN=unit-test", provenanceEvent.getDetails());
    }
    // Assert received flow files.
    processSession.assertAllFlowFilesTransferred(Relationship.ANONYMOUS);
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(Relationship.ANONYMOUS);
    assertEquals(2, flowFiles.size());
    for (final MockFlowFile flowFile : flowFiles) {
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_HOST.key(), peer.getHost());
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_ADDRESS.key(), peer.getHost() + ":" + peer.getPort());
        flowFile.assertAttributeEquals("client-attr-1", "client-attr-1-value");
        flowFile.assertAttributeEquals("client-attr-2", "client-attr-2-value");
    }
}

Example 13 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestStandardProcessSession method testForkOneToOneReported.

@Test
public void testForkOneToOneReported() throws IOException {
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord);
    // we have to increment the ID generator because we are creating a FlowFile without the FlowFile Repository's knowledge
    flowFileRepo.idGenerator.getAndIncrement();
    final FlowFile orig = session.get();
    final FlowFile newFlowFile = session.create(orig);
    session.transfer(newFlowFile, new Relationship.Builder().name("A").build());
    session.getProvenanceReporter().fork(newFlowFile, Collections.singleton(orig));
    session.remove(orig);
    session.commit();
    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 1000);
    assertEquals(2, events.size());
    final ProvenanceEventRecord firstRecord = events.get(0);
    final ProvenanceEventRecord secondRecord = events.get(1);
    assertEquals(ProvenanceEventType.FORK, firstRecord.getEventType());
    assertEquals(ProvenanceEventType.DROP, secondRecord.getEventType());
}

Example 14 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestStandardProcessSession method testContentModifiedEmittedAndNotAttributesModified.

@Test
public void testContentModifiedEmittedAndNotAttributesModified() throws IOException {
    final FlowFileRecord flowFile = new StandardFlowFileRecord.Builder().id(1L).addAttribute("uuid", "000000000000-0000-0000-0000-00000000").build();
    this.flowFileQueue.put(flowFile);
    FlowFile existingFlowFile = session.get();
    existingFlowFile = session.write(existingFlowFile, new OutputStreamCallback() {

        @Override
        public void process(OutputStream out) throws IOException {
        }
    });
    existingFlowFile = session.putAttribute(existingFlowFile, "attr", "a");
    session.transfer(existingFlowFile, new Relationship.Builder().name("A").build());
    session.commit();
    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 10000);
    assertFalse(events.isEmpty());
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals(ProvenanceEventType.CONTENT_MODIFIED, event.getEventType());
}

Example 15 with ProvenanceEventRecord

use of org.apache.nifi.provenance.ProvenanceEventRecord in project nifi by apache.

the class TestStandardProcessSession method testContentModifiedNotEmittedForCreate.

@Test
public void testContentModifiedNotEmittedForCreate() throws IOException {
    FlowFile newFlowFile = session.create();
    newFlowFile = session.write(newFlowFile, new OutputStreamCallback() {

        @Override
        public void process(OutputStream out) throws IOException {
        }
    });
    session.transfer(newFlowFile, new Relationship.Builder().name("A").build());
    session.commit();
    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 10000);
    assertFalse(events.isEmpty());
    assertEquals(1, events.size());
    final ProvenanceEventRecord event = events.get(0);
    assertEquals(ProvenanceEventType.CREATE, event.getEventType());
}