Examples with VersionControlInformationDTO org.apache.nifi.web.api.dto.VersionControlInformationDTO used on opensource projects

Example 11 with VersionControlInformationDTO

use of org.apache.nifi.web.api.dto.VersionControlInformationDTO in project nifi by apache.

the class SnippetUtils method findAllVersionControlInfo.

private static void findAllVersionControlInfo(final ProcessGroupDTO dto, final List<VersionControlInformationDTO> found) {
    final VersionControlInformationDTO vci = dto.getVersionControlInformation();
    if (vci != null) {
        found.add(vci);
    }
    final FlowSnippetDTO contents = dto.getContents();
    if (contents != null) {
        for (final ProcessGroupDTO child : contents.getProcessGroups()) {
            findAllVersionControlInfo(child, found);
        }
    }
}

Example 12 with VersionControlInformationDTO

use of org.apache.nifi.web.api.dto.VersionControlInformationDTO in project nifi by apache.

the class SnippetUtils method verifyNoDuplicateVersionControlInfoDtos.

private static void verifyNoDuplicateVersionControlInfoDtos(final ProcessGroup group, final Collection<VersionControlInformationDTO> snippetVcis) {
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null) {
        for (final VersionControlInformationDTO snippetVci : snippetVcis) {
            if (vci.getBucketIdentifier().equals(snippetVci.getBucketId()) && vci.getFlowIdentifier().equals(snippetVci.getFlowId())) {
                throw new IllegalArgumentException("Cannot place the given Process Group into the desired destination because the destination group or one of its ancestor groups is " + "under Version Control and one of the selected Process Groups is also under Version Control with the same Flow. A Process Group that is under Version Control " + "cannot contain a child Process Group that points to the same Versioned Flow.");
            }
        }
    }
    final ProcessGroup parent = group.getParent();
    if (parent != null) {
        verifyNoDuplicateVersionControlInfoDtos(parent, snippetVcis);
    }
}

Example 13 with VersionControlInformationDTO

use of org.apache.nifi.web.api.dto.VersionControlInformationDTO in project nifi by apache.

the class StandardFlowSynchronizer method addProcessGroup.

private ProcessGroup addProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, final StringEncryptor encryptor, final FlowEncodingVersion encodingVersion) throws ProcessorInstantiationException {
    // get the parent group ID
    final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier();
    // add the process group
    final ProcessGroupDTO processGroupDTO = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion);
    final ProcessGroup processGroup = controller.createProcessGroup(processGroupDTO.getId());
    processGroup.setComments(processGroupDTO.getComments());
    processGroup.setVersionedComponentId(processGroupDTO.getVersionedComponentId());
    processGroup.setPosition(toPosition(processGroupDTO.getPosition()));
    processGroup.setName(processGroupDTO.getName());
    processGroup.setParent(parentGroup);
    if (parentGroup == null) {
        controller.setRootGroup(processGroup);
    } else {
        parentGroup.addProcessGroup(processGroup);
    }
    // Set the variables for the variable registry
    final Map<String, String> variables = new HashMap<>();
    final List<Element> variableElements = getChildrenByTagName(processGroupElement, "variable");
    for (final Element variableElement : variableElements) {
        final String variableName = variableElement.getAttribute("name");
        final String variableValue = variableElement.getAttribute("value");
        if (variableName == null || variableValue == null) {
            continue;
        }
        variables.put(variableName, variableValue);
    }
    processGroup.setVariables(variables);
    final VersionControlInformationDTO versionControlInfoDto = processGroupDTO.getVersionControlInformation();
    if (versionControlInfoDto != null) {
        final FlowRegistry flowRegistry = controller.getFlowRegistryClient().getFlowRegistry(versionControlInfoDto.getRegistryId());
        final String registryName = flowRegistry == null ? versionControlInfoDto.getRegistryId() : flowRegistry.getName();
        versionControlInfoDto.setState(VersionedFlowState.SYNC_FAILURE.name());
        versionControlInfoDto.setStateExplanation("Process Group has not yet been synchronized with the Flow Registry");
        final StandardVersionControlInformation versionControlInformation = StandardVersionControlInformation.Builder.fromDto(versionControlInfoDto).registryName(registryName).build();
        // pass empty map for the version control mapping because the VersionedComponentId has already been set on the components
        processGroup.setVersionControlInformation(versionControlInformation, Collections.emptyMap());
    }
    // Add Controller Services
    final List<Element> serviceNodeList = getChildrenByTagName(processGroupElement, "controllerService");
    if (!serviceNodeList.isEmpty()) {
        final Map<ControllerServiceNode, Element> controllerServices = ControllerServiceLoader.loadControllerServices(serviceNodeList, controller, processGroup, encryptor);
        ControllerServiceLoader.enableControllerServices(controllerServices, controller, encryptor, autoResumeState);
    }
    // add processors
    final List<Element> processorNodeList = getChildrenByTagName(processGroupElement, "processor");
    for (final Element processorElement : processorNodeList) {
        final ProcessorDTO processorDTO = FlowFromDOMFactory.getProcessor(processorElement, encryptor);
        BundleCoordinate coordinate;
        try {
            coordinate = BundleUtils.getCompatibleBundle(processorDTO.getType(), processorDTO.getBundle());
        } catch (final IllegalStateException e) {
            final BundleDTO bundleDTO = processorDTO.getBundle();
            if (bundleDTO == null) {
                coordinate = BundleCoordinate.UNKNOWN_COORDINATE;
            } else {
                coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion());
            }
        }
        final ProcessorNode procNode = controller.createProcessor(processorDTO.getType(), processorDTO.getId(), coordinate, false);
        procNode.setVersionedComponentId(processorDTO.getVersionedComponentId());
        processGroup.addProcessor(procNode);
        updateProcessor(procNode, processorDTO, processGroup, controller);
    }
    // add input ports
    final List<Element> inputPortNodeList = getChildrenByTagName(processGroupElement, "inputPort");
    for (final Element inputPortElement : inputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(inputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteInputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalInputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addInputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableInputPort(port);
        }
    }
    // add output ports
    final List<Element> outputPortNodeList = getChildrenByTagName(processGroupElement, "outputPort");
    for (final Element outputPortElement : outputPortNodeList) {
        final PortDTO portDTO = FlowFromDOMFactory.getPort(outputPortElement);
        final Port port;
        if (processGroup.isRootGroup()) {
            port = controller.createRemoteOutputPort(portDTO.getId(), portDTO.getName());
        } else {
            port = controller.createLocalOutputPort(portDTO.getId(), portDTO.getName());
        }
        port.setVersionedComponentId(portDTO.getVersionedComponentId());
        port.setPosition(toPosition(portDTO.getPosition()));
        port.setComments(portDTO.getComments());
        port.setProcessGroup(processGroup);
        final Set<String> userControls = portDTO.getUserAccessControl();
        if (userControls != null && !userControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setUserAccessControl(userControls);
        }
        final Set<String> groupControls = portDTO.getGroupAccessControl();
        if (groupControls != null && !groupControls.isEmpty()) {
            if (!(port instanceof RootGroupPort)) {
                throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort");
            }
            ((RootGroupPort) port).setGroupAccessControl(groupControls);
        }
        processGroup.addOutputPort(port);
        if (portDTO.getConcurrentlySchedulableTaskCount() != null) {
            port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount());
        }
        final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState());
        if (ScheduledState.RUNNING.equals(scheduledState)) {
            controller.startConnectable(port);
        } else if (ScheduledState.DISABLED.equals(scheduledState)) {
            processGroup.disableOutputPort(port);
        }
    }
    // add funnels
    final List<Element> funnelNodeList = getChildrenByTagName(processGroupElement, "funnel");
    for (final Element funnelElement : funnelNodeList) {
        final FunnelDTO funnelDTO = FlowFromDOMFactory.getFunnel(funnelElement);
        final Funnel funnel = controller.createFunnel(funnelDTO.getId());
        funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId());
        funnel.setPosition(toPosition(funnelDTO.getPosition()));
        // Since this is called during startup, we want to add the funnel without enabling it
        // and then tell the controller to enable it. This way, if the controller is not fully
        // initialized, the starting of the funnel is delayed until the controller is ready.
        processGroup.addFunnel(funnel, false);
        controller.startConnectable(funnel);
    }
    // add labels
    final List<Element> labelNodeList = getChildrenByTagName(processGroupElement, "label");
    for (final Element labelElement : labelNodeList) {
        final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement);
        final Label label = controller.createLabel(labelDTO.getId(), labelDTO.getLabel());
        label.setVersionedComponentId(labelDTO.getVersionedComponentId());
        label.setStyle(labelDTO.getStyle());
        label.setPosition(toPosition(labelDTO.getPosition()));
        label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight()));
        processGroup.addLabel(label);
    }
    // add nested process groups (recursively)
    final List<Element> nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup");
    for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) {
        addProcessGroup(controller, processGroup, nestedProcessGroupElement, encryptor, encodingVersion);
    }
    // add remote process group
    final List<Element> remoteProcessGroupNodeList = getChildrenByTagName(processGroupElement, "remoteProcessGroup");
    for (final Element remoteProcessGroupElement : remoteProcessGroupNodeList) {
        final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteProcessGroupElement, encryptor);
        final RemoteProcessGroup remoteGroup = controller.createRemoteProcessGroup(remoteGroupDto.getId(), remoteGroupDto.getTargetUris());
        remoteGroup.setVersionedComponentId(remoteGroupDto.getVersionedComponentId());
        remoteGroup.setComments(remoteGroupDto.getComments());
        remoteGroup.setPosition(toPosition(remoteGroupDto.getPosition()));
        final String name = remoteGroupDto.getName();
        if (name != null && !name.trim().isEmpty()) {
            remoteGroup.setName(name);
        }
        remoteGroup.setProcessGroup(processGroup);
        remoteGroup.setCommunicationsTimeout(remoteGroupDto.getCommunicationsTimeout());
        if (remoteGroupDto.getYieldDuration() != null) {
            remoteGroup.setYieldDuration(remoteGroupDto.getYieldDuration());
        }
        final String transportProtocol = remoteGroupDto.getTransportProtocol();
        if (transportProtocol != null && !transportProtocol.trim().isEmpty()) {
            remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(transportProtocol.toUpperCase()));
        }
        if (remoteGroupDto.getProxyHost() != null) {
            remoteGroup.setProxyHost(remoteGroupDto.getProxyHost());
        }
        if (remoteGroupDto.getProxyPort() != null) {
            remoteGroup.setProxyPort(remoteGroupDto.getProxyPort());
        }
        if (remoteGroupDto.getProxyUser() != null) {
            remoteGroup.setProxyUser(remoteGroupDto.getProxyUser());
        }
        if (remoteGroupDto.getProxyPassword() != null) {
            remoteGroup.setProxyPassword(remoteGroupDto.getProxyPassword());
        }
        if (StringUtils.isBlank(remoteGroupDto.getLocalNetworkInterface())) {
            remoteGroup.setNetworkInterface(null);
        } else {
            remoteGroup.setNetworkInterface(remoteGroupDto.getLocalNetworkInterface());
        }
        final Set<RemoteProcessGroupPortDescriptor> inputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "inputPort")) {
            inputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setInputPorts(inputPorts, false);
        final Set<RemoteProcessGroupPortDescriptor> outputPorts = new HashSet<>();
        for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "outputPort")) {
            outputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement));
        }
        remoteGroup.setOutputPorts(outputPorts, false);
        processGroup.addRemoteProcessGroup(remoteGroup);
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : outputPorts) {
            final RemoteGroupPort port = remoteGroup.getOutputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
        for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : inputPorts) {
            final RemoteGroupPort port = remoteGroup.getInputPort(remoteGroupPortDTO.getId());
            if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) {
                controller.startTransmitting(port);
            }
        }
    }
    // add connections
    final List<Element> connectionNodeList = getChildrenByTagName(processGroupElement, "connection");
    for (final Element connectionElement : connectionNodeList) {
        final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement);
        final Connectable source;
        final ConnectableDTO sourceDto = dto.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(sourceDto.getGroupId());
            source = remoteGroup.getOutputPort(sourceDto.getId());
        } else {
            final ProcessGroup sourceGroup = controller.getGroup(sourceDto.getGroupId());
            if (sourceGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Source: " + dto.getSource().getGroupId());
            }
            source = sourceGroup.getConnectable(sourceDto.getId());
        }
        if (source == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Source: " + dto.getSource().getId());
        }
        final Connectable destination;
        final ConnectableDTO destinationDto = dto.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDto.getType())) {
            final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(destinationDto.getGroupId());
            destination = remoteGroup.getInputPort(destinationDto.getId());
        } else {
            final ProcessGroup destinationGroup = controller.getGroup(destinationDto.getGroupId());
            if (destinationGroup == null) {
                throw new RuntimeException("Found Invalid ProcessGroup ID for Destination: " + dto.getDestination().getGroupId());
            }
            destination = destinationGroup.getConnectable(destinationDto.getId());
        }
        if (destination == null) {
            throw new RuntimeException("Found Invalid Connectable ID for Destination: " + dto.getDestination().getId());
        }
        final Connection connection = controller.createConnection(dto.getId(), dto.getName(), source, destination, dto.getSelectedRelationships());
        connection.setVersionedComponentId(dto.getVersionedComponentId());
        connection.setProcessGroup(processGroup);
        final List<Position> bendPoints = new ArrayList<>();
        for (final PositionDTO bend : dto.getBends()) {
            bendPoints.add(new Position(bend.getX(), bend.getY()));
        }
        connection.setBendPoints(bendPoints);
        final Long zIndex = dto.getzIndex();
        if (zIndex != null) {
            connection.setZIndex(zIndex);
        }
        if (dto.getLabelIndex() != null) {
            connection.setLabelIndex(dto.getLabelIndex());
        }
        List<FlowFilePrioritizer> newPrioritizers = null;
        final List<String> prioritizers = dto.getPrioritizers();
        if (prioritizers != null) {
            final List<String> newPrioritizersClasses = new ArrayList<>(prioritizers);
            newPrioritizers = new ArrayList<>();
            for (final String className : newPrioritizersClasses) {
                try {
                    newPrioritizers.add(controller.createPrioritizer(className));
                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
                    throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e);
                }
            }
        }
        if (newPrioritizers != null) {
            connection.getFlowFileQueue().setPriorities(newPrioritizers);
        }
        if (dto.getBackPressureObjectThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold());
        }
        if (dto.getBackPressureDataSizeThreshold() != null) {
            connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold());
        }
        if (dto.getFlowFileExpiration() != null) {
            connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration());
        }
        processGroup.addConnection(connection);
    }
    final List<Element> templateNodeList = getChildrenByTagName(processGroupElement, "template");
    for (final Element templateNode : templateNodeList) {
        final TemplateDTO templateDTO = TemplateUtils.parseDto(templateNode);
        final Template template = new Template(templateDTO);
        processGroup.addTemplate(template);
    }
    return processGroup;
}

Example 14 with VersionControlInformationDTO

use of org.apache.nifi.web.api.dto.VersionControlInformationDTO in project nifi by apache.

the class StandardNiFiServiceFacade method setVersionControlInformation.

@Override
public VersionControlInformationEntity setVersionControlInformation(final Revision revision, final String processGroupId, final VersionControlInformationDTO versionControlInfo, final Map<String, String> versionedComponentMapping) {
    final ProcessGroup group = processGroupDAO.getProcessGroup(processGroupId);
    final RevisionUpdate<VersionControlInformationDTO> snapshot = updateComponent(revision, group, () -> processGroupDAO.updateVersionControlInformation(versionControlInfo, versionedComponentMapping), processGroup -> dtoFactory.createVersionControlInformationDto(processGroup));
    return entityFactory.createVersionControlInformationEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()));
}

Example 15 with VersionControlInformationDTO

use of org.apache.nifi.web.api.dto.VersionControlInformationDTO in project nifi by apache.

the class StandardNiFiServiceFacade method verifyImportProcessGroup.

private void verifyImportProcessGroup(final VersionControlInformationDTO vciDto, final VersionedProcessGroup contents, final ProcessGroup group) {
    if (group == null) {
        return;
    }
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null) {
        // that point to the same server (one could point to localhost while another points to 127.0.0.1, for instance)..
        if (Objects.equals(vciDto.getBucketId(), vci.getBucketIdentifier()) && Objects.equals(vciDto.getFlowId(), vci.getFlowIdentifier())) {
            throw new IllegalStateException("Cannot import the specified Versioned Flow into the Process Group because doing so would cause a recursive dataflow. " + "If Process Group A contains Process Group B, then Process Group B is not allowed to contain the flow identified by Process Group A.");
        }
    }
    final Set<VersionedProcessGroup> childGroups = contents.getProcessGroups();
    if (childGroups != null) {
        for (final VersionedProcessGroup childGroup : childGroups) {
            final VersionedFlowCoordinates childCoordinates = childGroup.getVersionedFlowCoordinates();
            if (childCoordinates != null) {
                final VersionControlInformationDTO childVci = new VersionControlInformationDTO();
                childVci.setBucketId(childCoordinates.getBucketId());
                childVci.setFlowId(childCoordinates.getFlowId());
                verifyImportProcessGroup(childVci, childGroup, group);
            }
        }
    }
    verifyImportProcessGroup(vciDto, contents, group.getParent());
}