Search in sources :

Example 16 with Snippet

use of org.apache.nifi.controller.Snippet in project nifi by apache.

the class SnippetUtils method populateFlowSnippet.

/**
 * Populates the specified snippet and returns the details.
 *
 * @param snippet snippet
 * @param recurse recurse
 * @param includeControllerServices whether or not to include controller services in the flow snippet dto
 * @return snippet
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public FlowSnippetDTO populateFlowSnippet(final Snippet snippet, final boolean recurse, final boolean includeControllerServices, boolean removeInstanceId) {
    final FlowSnippetDTO snippetDto = new FlowSnippetDTO(removeInstanceId);
    final String groupId = snippet.getParentGroupId();
    final ProcessGroup processGroup = flowController.getGroup(groupId);
    // ensure the group could be found
    if (processGroup == null) {
        throw new IllegalStateException("The parent process group for this snippet could not be found.");
    }
    // We need to ensure that the Controller Services that are added get added to the proper group.
    // This can potentially get a little bit tricky. Consider this scenario:
    // We have a Process Group G1. Within Process Group G1 is a Controller Service C1.
    // Also within G1 is a child Process Group, G2. Within G2 is a child Process Group, G3.
    // Within G3 are two child Process Groups: G4 and G5. Within each of these children,
    // we have a Processor (P1, P2) that references the Controller Service C1, defined 3 levels above.
    // Now, we create a template that encompasses only Process Groups G4 and G5. We need to ensure
    // that the Controller Service C1 is included at the 'root' of the template so that those
    // Processors within G4 and G5 both have access to the same Controller Service. This can be drawn
    // out thus:
    // 
    // G1 -- C1
    // |
    // |
    // G2
    // |
    // |
    // G3
    // |  \
    // |   \
    // G4   G5
    // |    |
    // |    |
    // P1   P2
    // 
    // Both P1 and P2 reference C1.
    // 
    // In order to accomplish this, we maintain two collections. First, we keep a Set of all Controller Services that have
    // been added. If we add a new Controller Service to the set, then we know it hasn't been added anywhere in the Snippet.
    // In that case, we determine the service's group ID. In the flow described above, if we template just groups G4 and G5,
    // then we need to include the Controller Service defined at G1. So we also keep a Map of Group ID to controller services
    // in that group. If the ParentGroupId of a Controller Service is not in our snippet, then we instead update the parent
    // ParentGroupId to be that of our highest-level process group (in this case G3, as that's where the template is created)
    // and then add the controller services to that group (NOTE: here, when we say we change the group ID and add to that group,
    // we are talking only about the DTO objects that make up the snippet. We do not actually modify the Process Group or the
    // Controller Services in our flow themselves!)
    final Set<ControllerServiceDTO> allServicesReferenced = new HashSet<>();
    final Map<String, FlowSnippetDTO> contentsByGroup = new HashMap<>();
    contentsByGroup.put(processGroup.getIdentifier(), snippetDto);
    // add any processors
    final Set<ControllerServiceDTO> controllerServices = new HashSet<>();
    final Set<ProcessorDTO> processors = new LinkedHashSet<>();
    if (!snippet.getProcessors().isEmpty()) {
        for (final String processorId : snippet.getProcessors().keySet()) {
            final ProcessorNode processor = processGroup.getProcessor(processorId);
            if (processor == null) {
                throw new IllegalStateException("A processor in this snippet could not be found.");
            }
            processors.add(dtoFactory.createProcessorDto(processor));
            if (includeControllerServices) {
                // Include all referenced services that are not already included in this snippet.
                getControllerServices(processor.getProperties()).stream().filter(svc -> allServicesReferenced.add(svc)).forEach(svc -> {
                    final String svcGroupId = svc.getParentGroupId();
                    final String destinationGroupId = contentsByGroup.containsKey(svcGroupId) ? svcGroupId : processGroup.getIdentifier();
                    svc.setParentGroupId(destinationGroupId);
                    controllerServices.add(svc);
                });
            }
        }
    }
    // add any connections
    final Set<ConnectionDTO> connections = new LinkedHashSet<>();
    if (!snippet.getConnections().isEmpty()) {
        for (final String connectionId : snippet.getConnections().keySet()) {
            final Connection connection = processGroup.getConnection(connectionId);
            if (connection == null) {
                throw new IllegalStateException("A connection in this snippet could not be found.");
            }
            connections.add(dtoFactory.createConnectionDto(connection));
        }
    }
    // add any funnels
    final Set<FunnelDTO> funnels = new LinkedHashSet<>();
    if (!snippet.getFunnels().isEmpty()) {
        for (final String funnelId : snippet.getFunnels().keySet()) {
            final Funnel funnel = processGroup.getFunnel(funnelId);
            if (funnel == null) {
                throw new IllegalStateException("A funnel in this snippet could not be found.");
            }
            funnels.add(dtoFactory.createFunnelDto(funnel));
        }
    }
    // add any input ports
    final Set<PortDTO> inputPorts = new LinkedHashSet<>();
    if (!snippet.getInputPorts().isEmpty()) {
        for (final String inputPortId : snippet.getInputPorts().keySet()) {
            final Port inputPort = processGroup.getInputPort(inputPortId);
            if (inputPort == null) {
                throw new IllegalStateException("An input port in this snippet could not be found.");
            }
            inputPorts.add(dtoFactory.createPortDto(inputPort));
        }
    }
    // add any labels
    final Set<LabelDTO> labels = new LinkedHashSet<>();
    if (!snippet.getLabels().isEmpty()) {
        for (final String labelId : snippet.getLabels().keySet()) {
            final Label label = processGroup.getLabel(labelId);
            if (label == null) {
                throw new IllegalStateException("A label in this snippet could not be found.");
            }
            labels.add(dtoFactory.createLabelDto(label));
        }
    }
    // add any output ports
    final Set<PortDTO> outputPorts = new LinkedHashSet<>();
    if (!snippet.getOutputPorts().isEmpty()) {
        for (final String outputPortId : snippet.getOutputPorts().keySet()) {
            final Port outputPort = processGroup.getOutputPort(outputPortId);
            if (outputPort == null) {
                throw new IllegalStateException("An output port in this snippet could not be found.");
            }
            outputPorts.add(dtoFactory.createPortDto(outputPort));
        }
    }
    // add any process groups
    final Set<ProcessGroupDTO> processGroups = new LinkedHashSet<>();
    if (!snippet.getProcessGroups().isEmpty()) {
        for (final String childGroupId : snippet.getProcessGroups().keySet()) {
            final ProcessGroup childGroup = processGroup.getProcessGroup(childGroupId);
            if (childGroup == null) {
                throw new IllegalStateException("A process group in this snippet could not be found.");
            }
            final ProcessGroupDTO childGroupDto = dtoFactory.createProcessGroupDto(childGroup, recurse);
            processGroups.add(childGroupDto);
            // maintain a listing of visited groups starting with each group in the snippet. this is used to determine
            // whether a referenced controller service should be included in the resulting snippet. if the service is
            // defined at groupId or one of it's ancestors, its considered outside of this snippet and will only be included
            // when the includeControllerServices is set to true. this happens above when considering the processors in this snippet
            final Set<String> visitedGroupIds = new HashSet<>();
            addControllerServices(childGroup, childGroupDto, allServicesReferenced, includeControllerServices, visitedGroupIds, contentsByGroup, processGroup.getIdentifier());
        }
    }
    // add any remote process groups
    final Set<RemoteProcessGroupDTO> remoteProcessGroups = new LinkedHashSet<>();
    if (!snippet.getRemoteProcessGroups().isEmpty()) {
        for (final String remoteProcessGroupId : snippet.getRemoteProcessGroups().keySet()) {
            final RemoteProcessGroup remoteProcessGroup = processGroup.getRemoteProcessGroup(remoteProcessGroupId);
            if (remoteProcessGroup == null) {
                throw new IllegalStateException("A remote process group in this snippet could not be found.");
            }
            remoteProcessGroups.add(dtoFactory.createRemoteProcessGroupDto(remoteProcessGroup));
        }
    }
    // Normalize the coordinates based on the locations of the other components
    final List<? extends ComponentDTO> components = new ArrayList<>();
    components.addAll((Set) processors);
    components.addAll((Set) connections);
    components.addAll((Set) funnels);
    components.addAll((Set) inputPorts);
    components.addAll((Set) labels);
    components.addAll((Set) outputPorts);
    components.addAll((Set) processGroups);
    components.addAll((Set) remoteProcessGroups);
    normalizeCoordinates(components);
    Set<ControllerServiceDTO> updatedControllerServices = snippetDto.getControllerServices();
    if (updatedControllerServices == null) {
        updatedControllerServices = new HashSet<>();
    }
    updatedControllerServices.addAll(controllerServices);
    snippetDto.setControllerServices(updatedControllerServices);
    snippetDto.setProcessors(processors);
    snippetDto.setConnections(connections);
    snippetDto.setFunnels(funnels);
    snippetDto.setInputPorts(inputPorts);
    snippetDto.setLabels(labels);
    snippetDto.setOutputPorts(outputPorts);
    snippetDto.setProcessGroups(processGroups);
    snippetDto.setRemoteProcessGroups(remoteProcessGroups);
    return snippetDto;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) RemoteProcessGroupContentsDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO) ProcessGroup(org.apache.nifi.groups.ProcessGroup) ProcessorConfigDTO(org.apache.nifi.web.api.dto.ProcessorConfigDTO) ConnectableType(org.apache.nifi.connectable.ConnectableType) LoggerFactory(org.slf4j.LoggerFactory) Port(org.apache.nifi.connectable.Port) ConnectionDTO(org.apache.nifi.web.api.dto.ConnectionDTO) StringUtils(org.apache.commons.lang3.StringUtils) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) ResourceType(org.apache.nifi.authorization.resource.ResourceType) PositionDTO(org.apache.nifi.web.api.dto.PositionDTO) SecureRandom(java.security.SecureRandom) LabelDTO(org.apache.nifi.web.api.dto.LabelDTO) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) PropertyDescriptorDTO(org.apache.nifi.web.api.dto.PropertyDescriptorDTO) TenantEntity(org.apache.nifi.web.api.entity.TenantEntity) Map(java.util.Map) Connection(org.apache.nifi.connectable.Connection) FunnelDTO(org.apache.nifi.web.api.dto.FunnelDTO) ComponentIdGenerator(org.apache.nifi.util.ComponentIdGenerator) Label(org.apache.nifi.controller.label.Label) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) AccessPolicyDAO(org.apache.nifi.web.dao.AccessPolicyDAO) Collection(java.util.Collection) Set(java.util.Set) UUID(java.util.UUID) RemoteProcessGroupPortDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO) Snippet(org.apache.nifi.controller.Snippet) Collectors(java.util.stream.Collectors) ResourceFactory(org.apache.nifi.authorization.resource.ResourceFactory) FlowController(org.apache.nifi.controller.FlowController) StandardCharsets(java.nio.charset.StandardCharsets) PortDTO(org.apache.nifi.web.api.dto.PortDTO) List(java.util.List) ScheduledState(org.apache.nifi.controller.ScheduledState) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) ProcessorDTO(org.apache.nifi.web.api.dto.ProcessorDTO) Entry(java.util.Map.Entry) ControllerServiceState(org.apache.nifi.controller.service.ControllerServiceState) DtoFactory(org.apache.nifi.web.api.dto.DtoFactory) Resource(org.apache.nifi.authorization.Resource) FlowSnippetDTO(org.apache.nifi.web.api.dto.FlowSnippetDTO) RemoteProcessGroupDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupDTO) ProcessorNode(org.apache.nifi.controller.ProcessorNode) Funnel(org.apache.nifi.connectable.Funnel) ControllerServiceNode(org.apache.nifi.controller.service.ControllerServiceNode) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) AccessPolicyDTO(org.apache.nifi.web.api.dto.AccessPolicyDTO) LinkedHashSet(java.util.LinkedHashSet) Logger(org.slf4j.Logger) RequestAction(org.apache.nifi.authorization.RequestAction) ComponentDTO(org.apache.nifi.web.api.dto.ComponentDTO) AccessPolicy(org.apache.nifi.authorization.AccessPolicy) Collections(java.util.Collections) ConnectableDTO(org.apache.nifi.web.api.dto.ConnectableDTO) Funnel(org.apache.nifi.connectable.Funnel) FlowSnippetDTO(org.apache.nifi.web.api.dto.FlowSnippetDTO) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) HashMap(java.util.HashMap) ConnectionDTO(org.apache.nifi.web.api.dto.ConnectionDTO) Port(org.apache.nifi.connectable.Port) Label(org.apache.nifi.controller.label.Label) ArrayList(java.util.ArrayList) ProcessorNode(org.apache.nifi.controller.ProcessorNode) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) RemoteProcessGroupDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupDTO) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) RemoteProcessGroupPortDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO) PortDTO(org.apache.nifi.web.api.dto.PortDTO) Connection(org.apache.nifi.connectable.Connection) RemoteProcessGroupDTO(org.apache.nifi.web.api.dto.RemoteProcessGroupDTO) FunnelDTO(org.apache.nifi.web.api.dto.FunnelDTO) ProcessorDTO(org.apache.nifi.web.api.dto.ProcessorDTO) ProcessGroup(org.apache.nifi.groups.ProcessGroup) RemoteProcessGroup(org.apache.nifi.groups.RemoteProcessGroup) LabelDTO(org.apache.nifi.web.api.dto.LabelDTO)

Aggregations

Snippet (org.apache.nifi.controller.Snippet)16 ProcessGroup (org.apache.nifi.groups.ProcessGroup)14 HashSet (java.util.HashSet)9 Connection (org.apache.nifi.connectable.Connection)9 Port (org.apache.nifi.connectable.Port)9 ProcessorNode (org.apache.nifi.controller.ProcessorNode)9 RemoteProcessGroup (org.apache.nifi.groups.RemoteProcessGroup)9 FlowSnippetDTO (org.apache.nifi.web.api.dto.FlowSnippetDTO)9 ArrayList (java.util.ArrayList)8 Funnel (org.apache.nifi.connectable.Funnel)8 LinkedHashSet (java.util.LinkedHashSet)7 List (java.util.List)7 Set (java.util.Set)7 Collectors (java.util.stream.Collectors)7 Resource (org.apache.nifi.authorization.Resource)7 PropertyDescriptor (org.apache.nifi.components.PropertyDescriptor)7 ControllerServiceNode (org.apache.nifi.controller.service.ControllerServiceNode)7 Collection (java.util.Collection)6 Collections (java.util.Collections)6 HashMap (java.util.HashMap)6