Examples with Connection org.apache.nifi.connectable.Connection used on opensource projects

Example 91 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardConnectionDAO method createFlowFileListingRequest.

@Override
public ListFlowFileStatus createFlowFileListingRequest(String id, String listingRequestId) {
    final Connection connection = locateConnection(id);
    final FlowFileQueue queue = connection.getFlowFileQueue();
    // ensure we can list
    verifyList(queue);
    return queue.listFlowFiles(listingRequestId, 100);
}

Example 92 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardConnectionDAO method getFlowFile.

@Override
public FlowFileRecord getFlowFile(String id, String flowFileUuid) {
    try {
        final Connection connection = locateConnection(id);
        final FlowFileQueue queue = connection.getFlowFileQueue();
        final FlowFileRecord flowFile = queue.getFlowFile(flowFileUuid);
        if (flowFile == null) {
            throw new ResourceNotFoundException(String.format("The FlowFile with UUID %s is no longer in the active queue.", flowFileUuid));
        }
        // get the attributes and ensure appropriate access
        final Map<String, String> attributes = flowFile.getAttributes();
        final Authorizable dataAuthorizable = new DataAuthorizable(connection.getSourceAuthorizable());
        dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser(), attributes);
        return flowFile;
    } catch (final IOException ioe) {
        logger.error(String.format("Unable to get the flowfile (%s) at this time.", flowFileUuid), ioe);
        throw new IllegalStateException("Unable to get the FlowFile at this time.");
    }
}

Example 93 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardConnectionDAO method createConnection.

@Override
public Connection createConnection(final String groupId, final ConnectionDTO connectionDTO) {
    final ProcessGroup group = locateProcessGroup(flowController, groupId);
    if (isNotNull(connectionDTO.getParentGroupId()) && !flowController.areGroupsSame(connectionDTO.getParentGroupId(), groupId)) {
        throw new IllegalStateException("Cannot specify a different Parent Group ID than the Group to which the Connection is being added");
    }
    // get the source and destination connectables
    final ConnectableDTO sourceConnectableDTO = connectionDTO.getSource();
    final ConnectableDTO destinationConnectableDTO = connectionDTO.getDestination();
    // ensure both are specified
    if (sourceConnectableDTO == null || destinationConnectableDTO == null) {
        throw new IllegalArgumentException("Both source and destinations must be specified.");
    }
    // if the source/destination connectable's group id has not been set, its inferred to be the current group
    if (sourceConnectableDTO.getGroupId() == null) {
        sourceConnectableDTO.setGroupId(groupId);
    }
    if (destinationConnectableDTO.getGroupId() == null) {
        destinationConnectableDTO.setGroupId(groupId);
    }
    // validate the proposed configuration
    final List<String> validationErrors = validateProposedConfiguration(groupId, connectionDTO);
    // ensure there was no validation errors
    if (!validationErrors.isEmpty()) {
        throw new ValidationException(validationErrors);
    }
    // find the source
    final Connectable source;
    if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceConnectableDTO.getType())) {
        final ProcessGroup sourceParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = sourceParentGroup.getRemoteProcessGroup(sourceConnectableDTO.getGroupId());
        source = remoteProcessGroup.getOutputPort(sourceConnectableDTO.getId());
    } else {
        final ProcessGroup sourceGroup = locateProcessGroup(flowController, sourceConnectableDTO.getGroupId());
        source = sourceGroup.getConnectable(sourceConnectableDTO.getId());
    }
    // find the destination
    final Connectable destination;
    if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationConnectableDTO.getType())) {
        final ProcessGroup destinationParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = destinationParentGroup.getRemoteProcessGroup(destinationConnectableDTO.getGroupId());
        destination = remoteProcessGroup.getInputPort(destinationConnectableDTO.getId());
    } else {
        final ProcessGroup destinationGroup = locateProcessGroup(flowController, destinationConnectableDTO.getGroupId());
        destination = destinationGroup.getConnectable(destinationConnectableDTO.getId());
    }
    // determine the relationships
    final Set<String> relationships = new HashSet<>();
    if (isNotNull(connectionDTO.getSelectedRelationships())) {
        relationships.addAll(connectionDTO.getSelectedRelationships());
    }
    // create the connection
    final Connection connection = flowController.createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
    // configure the connection
    configureConnection(connection, connectionDTO);
    // add the connection to the group
    group.addConnection(connection);
    return connection;
}

Example 94 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class StandardConnectionDAO method createFlowFileDropRequest.

@Override
public DropFlowFileStatus createFlowFileDropRequest(String id, String dropRequestId) {
    final Connection connection = locateConnection(id);
    final FlowFileQueue queue = connection.getFlowFileQueue();
    final NiFiUser user = NiFiUserUtils.getNiFiUser();
    if (user == null) {
        throw new WebApplicationException(new Throwable("Unable to access details for current user."));
    }
    return queue.dropFlowFiles(dropRequestId, user.getIdentity());
}

Example 95 with Connection

use of org.apache.nifi.connectable.Connection in project nifi by apache.

the class SnippetUtils method populateFlowSnippet.

/**
 * Populates the specified snippet and returns the details.
 *
 * @param snippet snippet
 * @param recurse recurse
 * @param includeControllerServices whether or not to include controller services in the flow snippet dto
 * @return snippet
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public FlowSnippetDTO populateFlowSnippet(final Snippet snippet, final boolean recurse, final boolean includeControllerServices, boolean removeInstanceId) {
    final FlowSnippetDTO snippetDto = new FlowSnippetDTO(removeInstanceId);
    final String groupId = snippet.getParentGroupId();
    final ProcessGroup processGroup = flowController.getGroup(groupId);
    // ensure the group could be found
    if (processGroup == null) {
        throw new IllegalStateException("The parent process group for this snippet could not be found.");
    }
    // We need to ensure that the Controller Services that are added get added to the proper group.
    // This can potentially get a little bit tricky. Consider this scenario:
    // We have a Process Group G1. Within Process Group G1 is a Controller Service C1.
    // Also within G1 is a child Process Group, G2. Within G2 is a child Process Group, G3.
    // Within G3 are two child Process Groups: G4 and G5. Within each of these children,
    // we have a Processor (P1, P2) that references the Controller Service C1, defined 3 levels above.
    // Now, we create a template that encompasses only Process Groups G4 and G5. We need to ensure
    // that the Controller Service C1 is included at the 'root' of the template so that those
    // Processors within G4 and G5 both have access to the same Controller Service. This can be drawn
    // out thus:
    // 
    // G1 -- C1
    // |
    // |
    // G2
    // |
    // |
    // G3
    // |  \
    // |   \
    // G4   G5
    // |    |
    // |    |
    // P1   P2
    // 
    // Both P1 and P2 reference C1.
    // 
    // In order to accomplish this, we maintain two collections. First, we keep a Set of all Controller Services that have
    // been added. If we add a new Controller Service to the set, then we know it hasn't been added anywhere in the Snippet.
    // In that case, we determine the service's group ID. In the flow described above, if we template just groups G4 and G5,
    // then we need to include the Controller Service defined at G1. So we also keep a Map of Group ID to controller services
    // in that group. If the ParentGroupId of a Controller Service is not in our snippet, then we instead update the parent
    // ParentGroupId to be that of our highest-level process group (in this case G3, as that's where the template is created)
    // and then add the controller services to that group (NOTE: here, when we say we change the group ID and add to that group,
    // we are talking only about the DTO objects that make up the snippet. We do not actually modify the Process Group or the
    // Controller Services in our flow themselves!)
    final Set<ControllerServiceDTO> allServicesReferenced = new HashSet<>();
    final Map<String, FlowSnippetDTO> contentsByGroup = new HashMap<>();
    contentsByGroup.put(processGroup.getIdentifier(), snippetDto);
    // add any processors
    final Set<ControllerServiceDTO> controllerServices = new HashSet<>();
    final Set<ProcessorDTO> processors = new LinkedHashSet<>();
    if (!snippet.getProcessors().isEmpty()) {
        for (final String processorId : snippet.getProcessors().keySet()) {
            final ProcessorNode processor = processGroup.getProcessor(processorId);
            if (processor == null) {
                throw new IllegalStateException("A processor in this snippet could not be found.");
            }
            processors.add(dtoFactory.createProcessorDto(processor));
            if (includeControllerServices) {
                // Include all referenced services that are not already included in this snippet.
                getControllerServices(processor.getProperties()).stream().filter(svc -> allServicesReferenced.add(svc)).forEach(svc -> {
                    final String svcGroupId = svc.getParentGroupId();
                    final String destinationGroupId = contentsByGroup.containsKey(svcGroupId) ? svcGroupId : processGroup.getIdentifier();
                    svc.setParentGroupId(destinationGroupId);
                    controllerServices.add(svc);
                });
            }
        }
    }
    // add any connections
    final Set<ConnectionDTO> connections = new LinkedHashSet<>();
    if (!snippet.getConnections().isEmpty()) {
        for (final String connectionId : snippet.getConnections().keySet()) {
            final Connection connection = processGroup.getConnection(connectionId);
            if (connection == null) {
                throw new IllegalStateException("A connection in this snippet could not be found.");
            }
            connections.add(dtoFactory.createConnectionDto(connection));
        }
    }
    // add any funnels
    final Set<FunnelDTO> funnels = new LinkedHashSet<>();
    if (!snippet.getFunnels().isEmpty()) {
        for (final String funnelId : snippet.getFunnels().keySet()) {
            final Funnel funnel = processGroup.getFunnel(funnelId);
            if (funnel == null) {
                throw new IllegalStateException("A funnel in this snippet could not be found.");
            }
            funnels.add(dtoFactory.createFunnelDto(funnel));
        }
    }
    // add any input ports
    final Set<PortDTO> inputPorts = new LinkedHashSet<>();
    if (!snippet.getInputPorts().isEmpty()) {
        for (final String inputPortId : snippet.getInputPorts().keySet()) {
            final Port inputPort = processGroup.getInputPort(inputPortId);
            if (inputPort == null) {
                throw new IllegalStateException("An input port in this snippet could not be found.");
            }
            inputPorts.add(dtoFactory.createPortDto(inputPort));
        }
    }
    // add any labels
    final Set<LabelDTO> labels = new LinkedHashSet<>();
    if (!snippet.getLabels().isEmpty()) {
        for (final String labelId : snippet.getLabels().keySet()) {
            final Label label = processGroup.getLabel(labelId);
            if (label == null) {
                throw new IllegalStateException("A label in this snippet could not be found.");
            }
            labels.add(dtoFactory.createLabelDto(label));
        }
    }
    // add any output ports
    final Set<PortDTO> outputPorts = new LinkedHashSet<>();
    if (!snippet.getOutputPorts().isEmpty()) {
        for (final String outputPortId : snippet.getOutputPorts().keySet()) {
            final Port outputPort = processGroup.getOutputPort(outputPortId);
            if (outputPort == null) {
                throw new IllegalStateException("An output port in this snippet could not be found.");
            }
            outputPorts.add(dtoFactory.createPortDto(outputPort));
        }
    }
    // add any process groups
    final Set<ProcessGroupDTO> processGroups = new LinkedHashSet<>();
    if (!snippet.getProcessGroups().isEmpty()) {
        for (final String childGroupId : snippet.getProcessGroups().keySet()) {
            final ProcessGroup childGroup = processGroup.getProcessGroup(childGroupId);
            if (childGroup == null) {
                throw new IllegalStateException("A process group in this snippet could not be found.");
            }
            final ProcessGroupDTO childGroupDto = dtoFactory.createProcessGroupDto(childGroup, recurse);
            processGroups.add(childGroupDto);
            // maintain a listing of visited groups starting with each group in the snippet. this is used to determine
            // whether a referenced controller service should be included in the resulting snippet. if the service is
            // defined at groupId or one of it's ancestors, its considered outside of this snippet and will only be included
            // when the includeControllerServices is set to true. this happens above when considering the processors in this snippet
            final Set<String> visitedGroupIds = new HashSet<>();
            addControllerServices(childGroup, childGroupDto, allServicesReferenced, includeControllerServices, visitedGroupIds, contentsByGroup, processGroup.getIdentifier());
        }
    }
    // add any remote process groups
    final Set<RemoteProcessGroupDTO> remoteProcessGroups = new LinkedHashSet<>();
    if (!snippet.getRemoteProcessGroups().isEmpty()) {
        for (final String remoteProcessGroupId : snippet.getRemoteProcessGroups().keySet()) {
            final RemoteProcessGroup remoteProcessGroup = processGroup.getRemoteProcessGroup(remoteProcessGroupId);
            if (remoteProcessGroup == null) {
                throw new IllegalStateException("A remote process group in this snippet could not be found.");
            }
            remoteProcessGroups.add(dtoFactory.createRemoteProcessGroupDto(remoteProcessGroup));
        }
    }
    // Normalize the coordinates based on the locations of the other components
    final List<? extends ComponentDTO> components = new ArrayList<>();
    components.addAll((Set) processors);
    components.addAll((Set) connections);
    components.addAll((Set) funnels);
    components.addAll((Set) inputPorts);
    components.addAll((Set) labels);
    components.addAll((Set) outputPorts);
    components.addAll((Set) processGroups);
    components.addAll((Set) remoteProcessGroups);
    normalizeCoordinates(components);
    Set<ControllerServiceDTO> updatedControllerServices = snippetDto.getControllerServices();
    if (updatedControllerServices == null) {
        updatedControllerServices = new HashSet<>();
    }
    updatedControllerServices.addAll(controllerServices);
    snippetDto.setControllerServices(updatedControllerServices);
    snippetDto.setProcessors(processors);
    snippetDto.setConnections(connections);
    snippetDto.setFunnels(funnels);
    snippetDto.setInputPorts(inputPorts);
    snippetDto.setLabels(labels);
    snippetDto.setOutputPorts(outputPorts);
    snippetDto.setProcessGroups(processGroups);
    snippetDto.setRemoteProcessGroups(remoteProcessGroups);
    return snippetDto;
}