Examples with VersionControlInformation org.apache.nifi.registry.flow.VersionControlInformation used on opensource projects

Example 11 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class NiFiRegistryFlowMapper method mapGroup.

private InstantiatedVersionedProcessGroup mapGroup(final ProcessGroup group, final ControllerServiceProvider serviceLookup, final FlowRegistryClient registryClient, final boolean topLevel, final boolean mapDescendantVersionedFlows) {
    final InstantiatedVersionedProcessGroup versionedGroup = new InstantiatedVersionedProcessGroup(group.getIdentifier(), group.getProcessGroupIdentifier());
    versionedGroup.setIdentifier(getId(group.getVersionedComponentId(), group.getIdentifier()));
    versionedGroup.setGroupIdentifier(getGroupId(group.getProcessGroupIdentifier()));
    versionedGroup.setName(group.getName());
    versionedGroup.setComments(group.getComments());
    versionedGroup.setPosition(mapPosition(group.getPosition()));
    // only for a child group that is itself version controlled.
    if (!topLevel) {
        final VersionControlInformation versionControlInfo = group.getVersionControlInformation();
        if (versionControlInfo != null) {
            final VersionedFlowCoordinates coordinates = new VersionedFlowCoordinates();
            final String registryId = versionControlInfo.getRegistryIdentifier();
            final FlowRegistry registry = registryClient.getFlowRegistry(registryId);
            if (registry == null) {
                throw new IllegalStateException("Process Group refers to a Flow Registry with ID " + registryId + " but no Flow Registry exists with that ID. Cannot resolve to a URL.");
            }
            coordinates.setRegistryUrl(registry.getURL());
            coordinates.setBucketId(versionControlInfo.getBucketIdentifier());
            coordinates.setFlowId(versionControlInfo.getFlowIdentifier());
            coordinates.setVersion(versionControlInfo.getVersion());
            versionedGroup.setVersionedFlowCoordinates(coordinates);
            // Otherwise, we will not be able to lookup the port when connecting to it.
            for (final Port port : group.getInputPorts()) {
                getId(port.getVersionedComponentId(), port.getIdentifier());
            }
            for (final Port port : group.getOutputPorts()) {
                getId(port.getVersionedComponentId(), port.getIdentifier());
            }
            // because the contents are remotely managed and not part of the versioning of this Process Group
            if (!mapDescendantVersionedFlows) {
                return versionedGroup;
            }
        }
    }
    versionedGroup.setControllerServices(group.getControllerServices(false).stream().map(service -> mapControllerService(service, serviceLookup)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setFunnels(group.getFunnels().stream().map(this::mapFunnel).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setInputPorts(group.getInputPorts().stream().map(this::mapPort).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setOutputPorts(group.getOutputPorts().stream().map(this::mapPort).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setLabels(group.getLabels().stream().map(this::mapLabel).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setProcessors(group.getProcessors().stream().map(processor -> mapProcessor(processor, serviceLookup)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setRemoteProcessGroups(group.getRemoteProcessGroups().stream().map(this::mapRemoteProcessGroup).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setProcessGroups(group.getProcessGroups().stream().map(grp -> mapGroup(grp, serviceLookup, registryClient, false, mapDescendantVersionedFlows)).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setConnections(group.getConnections().stream().map(this::mapConnection).collect(Collectors.toCollection(LinkedHashSet::new)));
    versionedGroup.setVariables(group.getVariableRegistry().getVariableMap().entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().getName(), Map.Entry::getValue)));
    return versionedGroup;
}

Example 12 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class SnippetUtils method verifyNoVersionControlConflicts.

public static void verifyNoVersionControlConflicts(final Snippet snippet, final ProcessGroup parentGroup, final ProcessGroup destination) {
    if (snippet == null) {
        return;
    }
    if (snippet.getProcessGroups() == null) {
        return;
    }
    final List<VersionControlInformation> vcis = new ArrayList<>();
    for (final String groupId : snippet.getProcessGroups().keySet()) {
        final ProcessGroup group = parentGroup.getProcessGroup(groupId);
        if (group != null) {
            findAllVersionControlInfo(group, vcis);
        }
    }
    verifyNoDuplicateVersionControlInfo(destination, vcis);
}

Example 13 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class SnippetUtils method verifyNoDuplicateVersionControlInfoDtos.

private static void verifyNoDuplicateVersionControlInfoDtos(final ProcessGroup group, final Collection<VersionControlInformationDTO> snippetVcis) {
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null) {
        for (final VersionControlInformationDTO snippetVci : snippetVcis) {
            if (vci.getBucketIdentifier().equals(snippetVci.getBucketId()) && vci.getFlowIdentifier().equals(snippetVci.getFlowId())) {
                throw new IllegalArgumentException("Cannot place the given Process Group into the desired destination because the destination group or one of its ancestor groups is " + "under Version Control and one of the selected Process Groups is also under Version Control with the same Flow. A Process Group that is under Version Control " + "cannot contain a child Process Group that points to the same Versioned Flow.");
            }
        }
    }
    final ProcessGroup parent = group.getParent();
    if (parent != null) {
        verifyNoDuplicateVersionControlInfoDtos(parent, snippetVcis);
    }
}

Example 14 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class FlowController method instantiateSnippet.

private void instantiateSnippet(final ProcessGroup group, final FlowSnippetDTO dto, final boolean topLevel) throws ProcessorInstantiationException {
    writeLock.lock();
    try {
        validateSnippetContents(requireNonNull(group), dto);
        // 
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(controllerServiceDTO.getType(), controllerServiceDTO.getBundle());
            final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), bundleCoordinate, Collections.emptySet(), true);
            serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData());
            serviceNode.setComments(controllerServiceDTO.getComments());
            serviceNode.setName(controllerServiceDTO.getName());
            if (!topLevel) {
                serviceNode.setVersionedComponentId(controllerServiceDTO.getVersionedComponentId());
            }
            group.addControllerService(serviceNode);
        }
        // references another service.
        for (final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices()) {
            final String serviceId = controllerServiceDTO.getId();
            final ControllerServiceNode serviceNode = getControllerServiceNode(serviceId);
            serviceNode.setProperties(controllerServiceDTO.getProperties());
        }
        // 
        for (final LabelDTO labelDTO : dto.getLabels()) {
            final Label label = createLabel(labelDTO.getId(), labelDTO.getLabel());
            label.setPosition(toPosition(labelDTO.getPosition()));
            if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) {
                label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
            }
            label.setStyle(labelDTO.getStyle());
            if (!topLevel) {
                label.setVersionedComponentId(labelDTO.getVersionedComponentId());
            }
            group.addLabel(label);
        }
        // Instantiate the funnels
        for (final FunnelDTO funnelDTO : dto.getFunnels()) {
            final Funnel funnel = createFunnel(funnelDTO.getId());
            funnel.setPosition(toPosition(funnelDTO.getPosition()));
            if (!topLevel) {
                funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
            }
            group.addFunnel(funnel);
        }
        // 
        for (final PortDTO portDTO : dto.getInputPorts()) {
            final Port inputPort;
            if (group.isRootGroup()) {
                inputPort = createRemoteInputPort(portDTO.getId(), portDTO.getName());
                inputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                inputPort = createLocalInputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                inputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            inputPort.setPosition(toPosition(portDTO.getPosition()));
            inputPort.setProcessGroup(group);
            inputPort.setComments(portDTO.getComments());
            group.addInputPort(inputPort);
        }
        for (final PortDTO portDTO : dto.getOutputPorts()) {
            final Port outputPort;
            if (group.isRootGroup()) {
                outputPort = createRemoteOutputPort(portDTO.getId(), portDTO.getName());
                outputPort.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
                if (portDTO.getGroupAccessControl() != null) {
                    ((RootGroupPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
                }
                if (portDTO.getUserAccessControl() != null) {
                    ((RootGroupPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
                }
            } else {
                outputPort = createLocalOutputPort(portDTO.getId(), portDTO.getName());
            }
            if (!topLevel) {
                outputPort.setVersionedComponentId(portDTO.getVersionedComponentId());
            }
            outputPort.setPosition(toPosition(portDTO.getPosition()));
            outputPort.setProcessGroup(group);
            outputPort.setComments(portDTO.getComments());
            group.addOutputPort(outputPort);
        }
        // 
        for (final ProcessorDTO processorDTO : dto.getProcessors()) {
            final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(processorDTO.getType(), processorDTO.getBundle());
            final ProcessorNode procNode = createProcessor(processorDTO.getType(), processorDTO.getId(), bundleCoordinate);
            procNode.setPosition(toPosition(processorDTO.getPosition()));
            procNode.setProcessGroup(group);
            if (!topLevel) {
                procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
            }
            final ProcessorConfigDTO config = processorDTO.getConfig();
            procNode.setComments(config.getComments());
            if (config.isLossTolerant() != null) {
                procNode.setLossTolerant(config.isLossTolerant());
            }
            procNode.setName(processorDTO.getName());
            procNode.setYieldPeriod(config.getYieldDuration());
            procNode.setPenalizationPeriod(config.getPenaltyDuration());
            procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel()));
            procNode.setAnnotationData(config.getAnnotationData());
            procNode.setStyle(processorDTO.getStyle());
            if (config.getRunDurationMillis() != null) {
                procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS);
            }
            if (config.getSchedulingStrategy() != null) {
                procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy()));
            }
            if (config.getExecutionNode() != null) {
                procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode()));
            }
            if (processorDTO.getState().equals(ScheduledState.DISABLED.toString())) {
                procNode.disable();
            }
            // ensure that the scheduling strategy is set prior to these values
            procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount());
            procNode.setScheduldingPeriod(config.getSchedulingPeriod());
            final Set<Relationship> relationships = new HashSet<>();
            if (processorDTO.getRelationships() != null) {
                for (final RelationshipDTO rel : processorDTO.getRelationships()) {
                    if (rel.isAutoTerminate()) {
                        relationships.add(procNode.getRelationship(rel.getName()));
                    }
                }
                procNode.setAutoTerminatedRelationships(relationships);
            }
            if (config.getProperties() != null) {
                procNode.setProperties(config.getProperties());
            }
            group.addProcessor(procNode);
        }
        // 
        for (final RemoteProcessGroupDTO remoteGroupDTO : dto.getRemoteProcessGroups()) {
            final RemoteProcessGroup remoteGroup = createRemoteProcessGroup(remoteGroupDTO.getId(), remoteGroupDTO.getTargetUris());
            remoteGroup.setComments(remoteGroupDTO.getComments());
            remoteGroup.setPosition(toPosition(remoteGroupDTO.getPosition()));
            remoteGroup.setCommunicationsTimeout(remoteGroupDTO.getCommunicationsTimeout());
            remoteGroup.setYieldDuration(remoteGroupDTO.getYieldDuration());
            if (!topLevel) {
                remoteGroup.setVersionedComponentId(remoteGroupDTO.getVersionedComponentId());
            }
            if (remoteGroupDTO.getTransportProtocol() == null) {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.RAW);
            } else {
                remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(remoteGroupDTO.getTransportProtocol()));
            }
            remoteGroup.setProxyHost(remoteGroupDTO.getProxyHost());
            remoteGroup.setProxyPort(remoteGroupDTO.getProxyPort());
            remoteGroup.setProxyUser(remoteGroupDTO.getProxyUser());
            remoteGroup.setProxyPassword(remoteGroupDTO.getProxyPassword());
            remoteGroup.setProcessGroup(group);
            // set the input/output ports
            if (remoteGroupDTO.getContents() != null) {
                final RemoteProcessGroupContentsDTO contents = remoteGroupDTO.getContents();
                // ensure there are input ports
                if (contents.getInputPorts() != null) {
                    remoteGroup.setInputPorts(convertRemotePort(contents.getInputPorts()), false);
                }
                // ensure there are output ports
                if (contents.getOutputPorts() != null) {
                    remoteGroup.setOutputPorts(convertRemotePort(contents.getOutputPorts()), false);
                }
            }
            group.addRemoteProcessGroup(remoteGroup);
        }
        // 
        for (final ProcessGroupDTO groupDTO : dto.getProcessGroups()) {
            final ProcessGroup childGroup = createProcessGroup(groupDTO.getId());
            childGroup.setParent(group);
            childGroup.setPosition(toPosition(groupDTO.getPosition()));
            childGroup.setComments(groupDTO.getComments());
            childGroup.setName(groupDTO.getName());
            if (groupDTO.getVariables() != null) {
                childGroup.setVariables(groupDTO.getVariables());
            }
            // We do this only if this component is the child of a Versioned Component.
            if (!topLevel) {
                childGroup.setVersionedComponentId(groupDTO.getVersionedComponentId());
            }
            group.addProcessGroup(childGroup);
            final FlowSnippetDTO contents = groupDTO.getContents();
            // we want this to be recursive, so we will create a new template that contains only
            // the contents of this child group and recursively call ourselves.
            final FlowSnippetDTO childTemplateDTO = new FlowSnippetDTO();
            childTemplateDTO.setConnections(contents.getConnections());
            childTemplateDTO.setInputPorts(contents.getInputPorts());
            childTemplateDTO.setLabels(contents.getLabels());
            childTemplateDTO.setOutputPorts(contents.getOutputPorts());
            childTemplateDTO.setProcessGroups(contents.getProcessGroups());
            childTemplateDTO.setProcessors(contents.getProcessors());
            childTemplateDTO.setFunnels(contents.getFunnels());
            childTemplateDTO.setRemoteProcessGroups(contents.getRemoteProcessGroups());
            childTemplateDTO.setControllerServices(contents.getControllerServices());
            instantiateSnippet(childGroup, childTemplateDTO, false);
            if (groupDTO.getVersionControlInformation() != null) {
                final VersionControlInformation vci = StandardVersionControlInformation.Builder.fromDto(groupDTO.getVersionControlInformation()).build();
                childGroup.setVersionControlInformation(vci, Collections.emptyMap());
            }
        }
        // 
        for (final ConnectionDTO connectionDTO : dto.getConnections()) {
            final ConnectableDTO sourceDTO = connectionDTO.getSource();
            final ConnectableDTO destinationDTO = connectionDTO.getDestination();
            final Connectable source;
            final Connectable destination;
            // see if the source connectable is a remote port
            if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(sourceDTO.getGroupId());
                source = remoteGroup.getOutputPort(sourceDTO.getId());
            } else {
                final ProcessGroup sourceGroup = getConnectableParent(group, sourceDTO.getGroupId());
                source = sourceGroup.getConnectable(sourceDTO.getId());
            }
            // see if the destination connectable is a remote port
            if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDTO.getType())) {
                final RemoteProcessGroup remoteGroup = group.getRemoteProcessGroup(destinationDTO.getGroupId());
                destination = remoteGroup.getInputPort(destinationDTO.getId());
            } else {
                final ProcessGroup destinationGroup = getConnectableParent(group, destinationDTO.getGroupId());
                destination = destinationGroup.getConnectable(destinationDTO.getId());
            }
            // determine the selection relationships for this connection
            final Set<String> relationships = new HashSet<>();
            if (connectionDTO.getSelectedRelationships() != null) {
                relationships.addAll(connectionDTO.getSelectedRelationships());
            }
            final Connection connection = createConnection(connectionDTO.getId(), connectionDTO.getName(), source, destination, relationships);
            if (!topLevel) {
                connection.setVersionedComponentId(connectionDTO.getVersionedComponentId());
            }
            if (connectionDTO.getBends() != null) {
                final List<Position> bendPoints = new ArrayList<>();
                for (final PositionDTO bend : connectionDTO.getBends()) {
                    bendPoints.add(new Position(bend.getX(), bend.getY()));
                }
                connection.setBendPoints(bendPoints);
            }
            final FlowFileQueue queue = connection.getFlowFileQueue();
            queue.setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold());
            queue.setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold());
            queue.setFlowFileExpiration(connectionDTO.getFlowFileExpiration());
            final List<String> prioritizers = connectionDTO.getPrioritizers();
            if (prioritizers != null) {
                final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
                final List<FlowFilePrioritizer> newPrioritizers = new ArrayList<>();
                for (final String className : newPrioritizersClasses) {
                    try {
                        newPrioritizers.add(createPrioritizer(className));
                    } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                        throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                    }
                }
                queue.setPriorities(newPrioritizers);
            }
            connection.setProcessGroup(group);
            group.addConnection(connection);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 15 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class StandardNiFiServiceFacade method verifyImportProcessGroup.

private void verifyImportProcessGroup(final VersionControlInformationDTO vciDto, final VersionedProcessGroup contents, final ProcessGroup group) {
    if (group == null) {
        return;
    }
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null) {
        // that point to the same server (one could point to localhost while another points to 127.0.0.1, for instance)..
        if (Objects.equals(vciDto.getBucketId(), vci.getBucketIdentifier()) && Objects.equals(vciDto.getFlowId(), vci.getFlowIdentifier())) {
            throw new IllegalStateException("Cannot import the specified Versioned Flow into the Process Group because doing so would cause a recursive dataflow. " + "If Process Group A contains Process Group B, then Process Group B is not allowed to contain the flow identified by Process Group A.");
        }
    }
    final Set<VersionedProcessGroup> childGroups = contents.getProcessGroups();
    if (childGroups != null) {
        for (final VersionedProcessGroup childGroup : childGroups) {
            final VersionedFlowCoordinates childCoordinates = childGroup.getVersionedFlowCoordinates();
            if (childCoordinates != null) {
                final VersionControlInformationDTO childVci = new VersionControlInformationDTO();
                childVci.setBucketId(childCoordinates.getBucketId());
                childVci.setFlowId(childCoordinates.getFlowId());
                verifyImportProcessGroup(childVci, childGroup, group);
            }
        }
    }
    verifyImportProcessGroup(vciDto, contents, group.getParent());
}