Examples with FlowFileQueue org.apache.nifi.controller.queue.FlowFileQueue used on opensource projects

Example 11 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class TestSimpleSwapSerializerDeserializer method testRoundTripSerializeDeserialize.

@Test
public void testRoundTripSerializeDeserialize() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
    }
    final String queueId = "87bb99fe-412c-49f6-a441-d1b0af4e20b4";
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn(queueId);
    final String swapLocation = "target/testRoundTrip-" + queueId + ".swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    try {
        final SimpleSwapSerializer serializer = new SimpleSwapSerializer();
        try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
            serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
        }
        final SimpleSwapDeserializer deserializer = new SimpleSwapDeserializer();
        final SwapContents swappedIn;
        try (final FileInputStream fis = new FileInputStream(swapFile);
            final DataInputStream dis = new DataInputStream(fis)) {
            swappedIn = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
        }
        assertEquals(toSwap.size(), swappedIn.getFlowFiles().size());
        for (int i = 0; i < toSwap.size(); i++) {
            final FlowFileRecord pre = toSwap.get(i);
            final FlowFileRecord post = swappedIn.getFlowFiles().get(i);
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getAttributes(), post.getAttributes());
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getId(), post.getId());
            assertEquals(pre.getContentClaim(), post.getContentClaim());
            assertEquals(pre.getContentClaimOffset(), post.getContentClaimOffset());
            assertEquals(pre.getEntryDate(), post.getEntryDate());
            assertEquals(pre.getLastQueueDate(), post.getLastQueueDate());
            assertEquals(pre.getLineageStartDate(), post.getLineageStartDate());
            assertEquals(pre.getPenaltyExpirationMillis(), post.getPenaltyExpirationMillis());
        }
    } finally {
        Files.deleteIfExists(swapFile.toPath());
    }
}

Example 12 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class TestFileSystemSwapManager method testBackwardCompatible.

@Test
public void testBackwardCompatible() throws IOException {
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/old-swap-file.swap"));
        final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) {
        final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
        Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
        final FileSystemSwapManager swapManager = createSwapManager();
        final SwapContents swapContents = swapManager.peek("src/test/resources/old-swap-file.swap", flowFileQueue);
        final List<FlowFileRecord> records = swapContents.getFlowFiles();
        assertEquals(10000, records.size());
        for (final FlowFileRecord record : records) {
            assertEquals(4, record.getAttributes().size());
            assertEquals("value", record.getAttribute("key"));
        }
    }
}

Example 13 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class TestStandardProcessSession method createConnection.

@SuppressWarnings("unchecked")
private Connection createConnection(AtomicReference<FlowFileQueue> flowFileQueueReference) {
    final Connection connection = Mockito.mock(Connection.class);
    FlowFileQueue flowFileQueueFromReference = flowFileQueueReference.get();
    final FlowFileQueue localFlowFileQueue;
    if (flowFileQueueFromReference == null) {
        localFlowFileQueue = createFlowFileQueueSpy(connection);
        flowFileQueueReference.set(localFlowFileQueue);
    } else {
        localFlowFileQueue = flowFileQueueFromReference;
    }
    when(connection.getFlowFileQueue()).thenReturn(localFlowFileQueue);
    Mockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            localFlowFileQueue.put((FlowFileRecord) invocation.getArguments()[0]);
            return null;
        }
    }).when(connection).enqueue(Mockito.any(FlowFileRecord.class));
    Mockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            localFlowFileQueue.putAll((Collection<FlowFileRecord>) invocation.getArguments()[0]);
            return null;
        }
    }).when(connection).enqueue(Mockito.any(Collection.class));
    final Connectable dest = Mockito.mock(Connectable.class);
    when(connection.getDestination()).thenReturn(dest);
    when(connection.getSource()).thenReturn(dest);
    Mockito.doAnswer(new Answer<FlowFile>() {

        @Override
        public FlowFile answer(InvocationOnMock invocation) throws Throwable {
            return localFlowFileQueue.poll(invocation.getArgumentAt(0, Set.class));
        }
    }).when(connection).poll(any(Set.class));
    Mockito.doAnswer(new Answer<List<FlowFileRecord>>() {

        @Override
        public List<FlowFileRecord> answer(InvocationOnMock invocation) throws Throwable {
            return localFlowFileQueue.poll(invocation.getArgumentAt(0, FlowFileFilter.class), invocation.getArgumentAt(1, Set.class));
        }
    }).when(connection).poll(any(FlowFileFilter.class), any(Set.class));
    Mockito.when(connection.getIdentifier()).thenReturn("conn-uuid");
    return connection;
}

Example 14 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class TestStandardProcessSession method testRoundRobinAcrossConnectionsOnSessionGetWithCount.

@Test
@SuppressWarnings("unchecked")
public void testRoundRobinAcrossConnectionsOnSessionGetWithCount() {
    final AtomicReference<FlowFileQueue> queue1Reference = new AtomicReference<>();
    final AtomicReference<FlowFileQueue> queue2Reference = new AtomicReference<>();
    final List<Connection> connList = new ArrayList<>();
    final Connection conn1 = createConnection(queue1Reference);
    final Connection conn2 = createConnection(queue2Reference);
    connList.add(conn1);
    connList.add(conn2);
    final FlowFileQueue queue2 = queue2Reference.get();
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().id(1000L).addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    queue2.put(flowFileRecord);
    when(connectable.getIncomingConnections()).thenReturn(connList);
    List<FlowFile> result = session.get(2);
    assertEquals(1, result.size());
    verify(conn1, times(1)).poll(any(FlowFileFilter.class), any(Set.class));
    verify(conn2, times(1)).poll(any(FlowFileFilter.class), any(Set.class));
}

Example 15 with FlowFileQueue

use of org.apache.nifi.controller.queue.FlowFileQueue in project nifi by apache.

the class StandardProcessGroup method verifyCanUpdate.

@Override
public void verifyCanUpdate(final VersionedFlowSnapshot updatedFlow, final boolean verifyConnectionRemoval, final boolean verifyNotDirty) {
    readLock.lock();
    try {
        final VersionControlInformation versionControlInfo = getVersionControlInformation();
        if (versionControlInfo != null) {
            if (!versionControlInfo.getFlowIdentifier().equals(updatedFlow.getSnapshotMetadata().getFlowIdentifier())) {
                throw new IllegalStateException(this + " is under version control but the given flow does not match the flow that this Process Group is synchronized with");
            }
            if (verifyNotDirty) {
                final VersionedFlowState flowState = versionControlInfo.getStatus().getState();
                final boolean modified = flowState == VersionedFlowState.LOCALLY_MODIFIED || flowState == VersionedFlowState.LOCALLY_MODIFIED_AND_STALE;
                final Set<FlowDifference> modifications = getModifications();
                if (modified) {
                    final String changes = modifications.stream().map(FlowDifference::toString).collect(Collectors.joining("\n"));
                    LOG.error("Cannot change the Version of the flow for {} because the Process Group has been modified ({} modifications) " + "since it was last synchronized with the Flow Registry. The following differences were found:\n{}", this, modifications.size(), changes);
                    throw new IllegalStateException("Cannot change the Version of the flow for " + this + " because the Process Group has been modified (" + modifications.size() + " modifications) since it was last synchronized with the Flow Registry. The Process Group must be" + " reverted to its original form before changing the version.");
                }
            }
            verifyNoDescendantsWithLocalModifications("be updated");
        }
        final VersionedProcessGroup flowContents = updatedFlow.getFlowContents();
        if (verifyConnectionRemoval) {
            // Determine which Connections have been removed.
            final Map<String, Connection> removedConnectionByVersionedId = new HashMap<>();
            // Populate the 'removedConnectionByVersionId' map with all Connections. We key off of the connection's VersionedComponentID
            // if it is populated. Otherwise, we key off of its actual ID. We do this because it allows us to then remove from this Map
            // any connection that does exist in the proposed flow. This results in us having a Map whose values are those Connections
            // that were removed. We can then check for any connections that have data in them. If any Connection is to be removed but
            // has data, then we should throw an IllegalStateException.
            findAllConnections().stream().forEach(conn -> removedConnectionByVersionedId.put(conn.getVersionedComponentId().orElse(conn.getIdentifier()), conn));
            final Set<String> proposedFlowConnectionIds = new HashSet<>();
            findAllConnectionIds(flowContents, proposedFlowConnectionIds);
            for (final String proposedConnectionId : proposedFlowConnectionIds) {
                removedConnectionByVersionedId.remove(proposedConnectionId);
            }
            // If any connection that was removed has data in it, throw an IllegalStateException
            for (final Connection connection : removedConnectionByVersionedId.values()) {
                final FlowFileQueue flowFileQueue = connection.getFlowFileQueue();
                if (!flowFileQueue.isEmpty()) {
                    throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the " + "proposed version does not contain " + connection + " and the connection currently has data in the queue.");
                }
            }
        }
        // Determine which input ports were removed from this process group
        final Map<String, Port> removedInputPortsByVersionId = new HashMap<>();
        getInputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedInputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getInputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedInputPortsByVersionId.remove(id));
        // Ensure that there are no incoming connections for any Input Port that was removed.
        for (final Port inputPort : removedInputPortsByVersionId.values()) {
            final List<Connection> incomingConnections = inputPort.getIncomingConnections();
            if (!incomingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Input Port " + inputPort + " and the Input Port currently has an incoming connections");
            }
        }
        // Determine which output ports were removed from this process group
        final Map<String, Port> removedOutputPortsByVersionId = new HashMap<>();
        getOutputPorts().stream().filter(port -> port.getVersionedComponentId().isPresent()).forEach(port -> removedOutputPortsByVersionId.put(port.getVersionedComponentId().get(), port));
        flowContents.getOutputPorts().stream().map(VersionedPort::getIdentifier).forEach(id -> removedOutputPortsByVersionId.remove(id));
        // Ensure that there are no outgoing connections for any Output Port that was removed.
        for (final Port outputPort : removedOutputPortsByVersionId.values()) {
            final Set<Connection> outgoingConnections = outputPort.getConnections();
            if (!outgoingConnections.isEmpty()) {
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the proposed version does not contain the Output Port " + outputPort + " and the Output Port currently has an outgoing connections");
            }
        }
        // Find any Process Groups that may have been deleted. If we find any Process Group that was deleted, and that Process Group
        // has Templates, then we fail because the Templates have to be removed first.
        final Map<String, VersionedProcessGroup> proposedProcessGroups = new HashMap<>();
        findAllProcessGroups(updatedFlow.getFlowContents(), proposedProcessGroups);
        for (final ProcessGroup childGroup : findAllProcessGroups()) {
            if (childGroup.getTemplates().isEmpty()) {
                continue;
            }
            final Optional<String> versionedIdOption = childGroup.getVersionedComponentId();
            if (!versionedIdOption.isPresent()) {
                continue;
            }
            final String versionedId = versionedIdOption.get();
            if (!proposedProcessGroups.containsKey(versionedId)) {
                // Process Group was removed.
                throw new IllegalStateException(this + " cannot be updated to the proposed version of the flow because the child " + childGroup + " that exists locally has one or more Templates, and the proposed flow does not contain these templates. " + "A Process Group cannot be deleted while it contains Templates. Please remove the Templates before attempting to change the version of the flow.");
            }
        }
        // Ensure that all Processors are instantiate-able.
        final Map<String, VersionedProcessor> proposedProcessors = new HashMap<>();
        findAllProcessors(updatedFlow.getFlowContents(), proposedProcessors);
        findAllProcessors().stream().filter(proc -> proc.getVersionedComponentId().isPresent()).forEach(proc -> proposedProcessors.remove(proc.getVersionedComponentId().get()));
        for (final VersionedProcessor processorToAdd : proposedProcessors.values()) {
            final BundleCoordinate coordinate = toCoordinate(processorToAdd.getBundle());
            try {
                flowController.createProcessor(processorToAdd.getType(), UUID.randomUUID().toString(), coordinate, false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Processor of type " + processorToAdd.getType(), e);
            }
        }
        // Ensure that all Controller Services are instantiate-able.
        final Map<String, VersionedControllerService> proposedServices = new HashMap<>();
        findAllControllerServices(updatedFlow.getFlowContents(), proposedServices);
        findAllControllerServices().stream().filter(service -> service.getVersionedComponentId().isPresent()).forEach(service -> proposedServices.remove(service.getVersionedComponentId().get()));
        for (final VersionedControllerService serviceToAdd : proposedServices.values()) {
            final BundleCoordinate coordinate = toCoordinate(serviceToAdd.getBundle());
            try {
                flowController.createControllerService(serviceToAdd.getType(), UUID.randomUUID().toString(), coordinate, Collections.emptySet(), false);
            } catch (Exception e) {
                throw new IllegalArgumentException("Unable to create Controller Service of type " + serviceToAdd.getType(), e);
            }
        }
        // Ensure that all Prioritizers are instantiate-able.
        final Map<String, VersionedConnection> proposedConnections = new HashMap<>();
        findAllConnections(updatedFlow.getFlowContents(), proposedConnections);
        findAllConnections().stream().filter(conn -> conn.getVersionedComponentId().isPresent()).forEach(conn -> proposedConnections.remove(conn.getVersionedComponentId().get()));
        for (final VersionedConnection connectionToAdd : proposedConnections.values()) {
            if (connectionToAdd.getPrioritizers() != null) {
                for (final String prioritizerType : connectionToAdd.getPrioritizers()) {
                    try {
                        flowController.createPrioritizer(prioritizerType);
                    } catch (Exception e) {
                        throw new IllegalArgumentException("Unable to create Prioritizer of type " + prioritizerType, e);
                    }
                }
            }
        }
    } finally {
        readLock.unlock();
    }
}