Examples with MockFlowFile org.apache.nifi.util.MockFlowFile used on opensource projects

Example 6 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project kylo by Teradata.

the class MergeHiveTableMetadataTest method runProcessor.

private void runProcessor(InputStream testDocument) {
    TestRunner nifiTestRunner = TestRunners.newTestRunner(new MergeHiveTableMetadata());
    nifiTestRunner.setValidateExpressionUsage(true);
    nifiTestRunner.setProperty(MergeHiveTableMetadata.DATABASE_NAME, "NAME");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.DATABASE_OWNER, "OWNER_NAME");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.TABLE_CREATE_TIME, "CREATE_TIME");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.TABLE_NAME, "TBL_NAME");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.TABLE_TYPE, "TBL_TYPE");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.COLUMN_NAME, "COLUMN_NAME");
    nifiTestRunner.setProperty(MergeHiveTableMetadata.COLUMN_TYPE, "TYPE_NAME");
    nifiTestRunner.assertValid();
    nifiTestRunner.enqueue(testDocument, new HashMap<String, String>() {

        {
            put("doc_id", "8736522777");
        }
    });
    nifiTestRunner.run(1, true, true);
    nifiTestRunner.assertAllFlowFilesTransferred(IndexElasticSearch.REL_SUCCESS, 1);
    final MockFlowFile out = nifiTestRunner.getFlowFilesForRelationship(IndexElasticSearch.REL_SUCCESS).get(0);
    String outgoingJson = new String(out.toByteArray());
    assertNotNull(out);
    out.assertAttributeEquals("doc_id", "8736522777");
}

Example 7 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestWriteAheadFlowFileRepository method testUpdatePerformance.

@Test
@Ignore("Intended only for local performance testing before/after making changes")
public void testUpdatePerformance() throws IOException, InterruptedException {
    final FlowFileQueue queue = new FlowFileQueue() {

        @Override
        public String getIdentifier() {
            return "4444";
        }

        @Override
        public List<FlowFilePrioritizer> getPriorities() {
            return null;
        }

        @Override
        public SwapSummary recoverSwappedFlowFiles() {
            return null;
        }

        @Override
        public void purgeSwapFiles() {
        }

        @Override
        public int getSwapFileCount() {
            return 0;
        }

        @Override
        public void setPriorities(List<FlowFilePrioritizer> newPriorities) {
        }

        @Override
        public void setBackPressureObjectThreshold(long maxQueueSize) {
        }

        @Override
        public long getBackPressureObjectThreshold() {
            return 0;
        }

        @Override
        public void setBackPressureDataSizeThreshold(String maxDataSize) {
        }

        @Override
        public String getBackPressureDataSizeThreshold() {
            return null;
        }

        @Override
        public QueueSize size() {
            return null;
        }

        @Override
        public boolean isEmpty() {
            return false;
        }

        @Override
        public boolean isActiveQueueEmpty() {
            return false;
        }

        @Override
        public QueueSize getUnacknowledgedQueueSize() {
            return null;
        }

        @Override
        public QueueSize getActiveQueueSize() {
            return size();
        }

        @Override
        public QueueSize getSwapQueueSize() {
            return null;
        }

        @Override
        public void acknowledge(FlowFileRecord flowFile) {
        }

        @Override
        public void acknowledge(Collection<FlowFileRecord> flowFiles) {
        }

        @Override
        public boolean isAllActiveFlowFilesPenalized() {
            return false;
        }

        @Override
        public boolean isAnyActiveFlowFilePenalized() {
            return false;
        }

        @Override
        public boolean isFull() {
            return false;
        }

        @Override
        public void put(FlowFileRecord file) {
        }

        @Override
        public void putAll(Collection<FlowFileRecord> files) {
        }

        @Override
        public FlowFileRecord poll(Set<FlowFileRecord> expiredRecords) {
            return null;
        }

        @Override
        public List<FlowFileRecord> poll(int maxResults, Set<FlowFileRecord> expiredRecords) {
            return null;
        }

        @Override
        public long drainQueue(Queue<FlowFileRecord> sourceQueue, List<FlowFileRecord> destination, int maxResults, Set<FlowFileRecord> expiredRecords) {
            return 0;
        }

        @Override
        public List<FlowFileRecord> poll(FlowFileFilter filter, Set<FlowFileRecord> expiredRecords) {
            return null;
        }

        @Override
        public String getFlowFileExpiration() {
            return null;
        }

        @Override
        public int getFlowFileExpiration(TimeUnit timeUnit) {
            return 0;
        }

        @Override
        public void setFlowFileExpiration(String flowExpirationPeriod) {
        }

        @Override
        public DropFlowFileStatus dropFlowFiles(String requestIdentifier, String requestor) {
            return null;
        }

        @Override
        public DropFlowFileStatus getDropFlowFileStatus(String requestIdentifier) {
            return null;
        }

        @Override
        public DropFlowFileStatus cancelDropFlowFileRequest(String requestIdentifier) {
            return null;
        }

        @Override
        public ListFlowFileStatus listFlowFiles(String requestIdentifier, int maxResults) {
            return null;
        }

        @Override
        public ListFlowFileStatus getListFlowFileStatus(String requestIdentifier) {
            return null;
        }

        @Override
        public ListFlowFileStatus cancelListFlowFileRequest(String requestIdentifier) {
            return null;
        }

        @Override
        public FlowFileRecord getFlowFile(String flowFileUuid) throws IOException {
            return null;
        }

        @Override
        public void verifyCanList() throws IllegalStateException {
        }
    };
    final int numPartitions = 16;
    final int numThreads = 8;
    final int totalUpdates = 160_000_000;
    final int batchSize = 10;
    final Path path = Paths.get("target/minimal-locking-repo");
    deleteRecursively(path.toFile());
    assertTrue(path.toFile().mkdirs());
    final ResourceClaimManager claimManager = new StandardResourceClaimManager();
    final RepositoryRecordSerdeFactory serdeFactory = new RepositoryRecordSerdeFactory(claimManager);
    final WriteAheadRepository<RepositoryRecord> repo = new MinimalLockingWriteAheadLog<>(path, numPartitions, serdeFactory, null);
    final Collection<RepositoryRecord> initialRecs = repo.recoverRecords();
    assertTrue(initialRecs.isEmpty());
    final int updateCountPerThread = totalUpdates / numThreads;
    final Thread[] threads = new Thread[numThreads];
    for (int j = 0; j < 2; j++) {
        for (int i = 0; i < numThreads; i++) {
            final Thread t = new Thread(new Runnable() {

                @Override
                public void run() {
                    final List<RepositoryRecord> records = new ArrayList<>();
                    final int numBatches = updateCountPerThread / batchSize;
                    final MockFlowFile baseFlowFile = new MockFlowFile(0L);
                    for (int i = 0; i < numBatches; i++) {
                        records.clear();
                        for (int k = 0; k < batchSize; k++) {
                            final FlowFileRecord flowFile = new MockFlowFile(i % 100_000, baseFlowFile);
                            final String uuid = flowFile.getAttribute("uuid");
                            final StandardRepositoryRecord record = new StandardRepositoryRecord(null, flowFile);
                            record.setDestination(queue);
                            final Map<String, String> updatedAttrs = Collections.singletonMap("uuid", uuid);
                            record.setWorking(flowFile, updatedAttrs);
                            records.add(record);
                        }
                        try {
                            repo.update(records, false);
                        } catch (IOException e) {
                            e.printStackTrace();
                            Assert.fail(e.toString());
                        }
                    }
                }
            });
            t.setDaemon(true);
            threads[i] = t;
        }
        final long start = System.nanoTime();
        for (final Thread t : threads) {
            t.start();
        }
        for (final Thread t : threads) {
            t.join();
        }
        final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
        if (j == 0) {
            System.out.println(millis + " ms to insert " + updateCountPerThread * numThreads + " updates using " + numPartitions + " partitions and " + numThreads + " threads, *as a warmup!*");
        } else {
            System.out.println(millis + " ms to insert " + updateCountPerThread * numThreads + " updates using " + numPartitions + " partitions and " + numThreads + " threads");
        }
    }
}

Example 8 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestStandardRemoteGroupPort method testSendRaw.

@Test
public void testSendRaw() throws Exception {
    setupMock(SiteToSiteTransportProtocol.RAW, TransferDirection.SEND);
    setupMockProcessSession();
    final String peerUrl = "nifi://node1.example.com:9090";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 9090, true);
    try (final SocketChannel socketChannel = SocketChannel.open()) {
        final CommunicationsSession commsSession = new SocketChannelCommunicationsSession(socketChannel);
        commsSession.setUserDn("nifi.node1.example.com");
        final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
        doReturn(peer).when(transaction).getCommunicant();
        final MockFlowFile flowFile = processSession.createFlowFile("0123456789".getBytes());
        sessionState.getFlowFileQueue().offer(flowFile);
        port.onTrigger(processContext, processSession);
        // Assert provenance.
        final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
        assertEquals(1, provenanceEvents.size());
        final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
        assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
        assertEquals(peerUrl + "/" + flowFile.getAttribute(CoreAttributes.UUID.key()), provenanceEvent.getTransitUri());
        assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
    }
}

Example 9 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestStandardRemoteGroupPort method testReceiveRaw.

@Test
public void testReceiveRaw() throws Exception {
    setupMock(SiteToSiteTransportProtocol.RAW, TransferDirection.RECEIVE);
    setupMockProcessSession();
    final String peerUrl = "nifi://node1.example.com:9090";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 9090, true);
    try (final SocketChannel socketChannel = SocketChannel.open()) {
        final CommunicationsSession commsSession = new SocketChannelCommunicationsSession(socketChannel);
        commsSession.setUserDn("nifi.node1.example.com");
        final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
        doReturn(peer).when(transaction).getCommunicant();
        final String sourceFlowFileUuid = "flowfile-uuid";
        final Map<String, String> attributes = new HashMap<>();
        attributes.put(CoreAttributes.UUID.key(), sourceFlowFileUuid);
        final byte[] dataPacketContents = "DataPacket Contents".getBytes();
        final ByteArrayInputStream dataPacketInputStream = new ByteArrayInputStream(dataPacketContents);
        final DataPacket dataPacket = new StandardDataPacket(attributes, dataPacketInputStream, dataPacketContents.length);
        // Return null when it gets called second time.
        doReturn(dataPacket).doReturn(null).when(this.transaction).receive();
        port.onTrigger(processContext, processSession);
        // Assert provenance.
        final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
        assertEquals(1, provenanceEvents.size());
        final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
        assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
        assertEquals(peerUrl + "/" + sourceFlowFileUuid, provenanceEvent.getTransitUri());
        assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
        // Assert received flow files.
        processSession.assertAllFlowFilesTransferred(Relationship.ANONYMOUS);
        final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(Relationship.ANONYMOUS);
        assertEquals(1, flowFiles.size());
        final MockFlowFile flowFile = flowFiles.get(0);
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_HOST.key(), peer.getHost());
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_ADDRESS.key(), peer.getHost() + ":" + peer.getPort());
    }
}

Example 10 with MockFlowFile

use of org.apache.nifi.util.MockFlowFile in project nifi by apache.

the class TestStandardRemoteGroupPort method testSendHttp.

@Test
public void testSendHttp() throws Exception {
    setupMock(SiteToSiteTransportProtocol.HTTP, TransferDirection.SEND);
    setupMockProcessSession();
    final String peerUrl = "https://node1.example.com:8080/nifi";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 8080, true);
    final HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    commsSession.setUserDn("nifi.node1.example.com");
    final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
    final String flowFileEndpointUri = "https://node1.example.com:8080/nifi-api/output-ports/port-id/transactions/transaction-id/flow-files";
    doReturn(peer).when(transaction).getCommunicant();
    commsSession.setDataTransferUrl(flowFileEndpointUri);
    final MockFlowFile flowFile = processSession.createFlowFile("0123456789".getBytes());
    sessionState.getFlowFileQueue().offer(flowFile);
    port.onTrigger(processContext, processSession);
    // Assert provenance.
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(1, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
    assertEquals(flowFileEndpointUri, provenanceEvent.getTransitUri());
    assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
}