Examples with ConnectionDTO org.apache.nifi.web.api.dto.ConnectionDTO used on opensource projects

Example 36 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class RemoteInputPortService method removeRemoteInputPorts.

public boolean removeRemoteInputPorts(RemoteInportPortRemovalData removalData) {
    Optional<TemplateRemoteInputPortConnections> remoteInputPortConnections = Optional.ofNullable(removalData.getConnectionsToRemove());
    if (!remoteInputPortConnections.isPresent()) {
        remoteInputPortConnections = getExistingRemoteProcessInputPortInformation(removalData.getTemplateName());
    }
    Set<String> inputPortNamesToRemove = removalData.getInputPortNamesToRemove();
    // Find the connections that match the input ports that are to be removed
    Set<ConnectionDTO> connectionsToRemove = remoteInputPortConnections.get().getExistingRemoteConnectionsToTemplate().stream().filter(connectionDTO -> inputPortNamesToRemove.contains(connectionDTO.getSource().getName())).collect(Collectors.toSet());
    Set<String> remoteInputPortIdsToRemove = connectionsToRemove.stream().map(connectionDTO -> connectionDTO.getSource().getId()).collect(Collectors.toSet());
    log.info("Removing input ports {}", inputPortNamesToRemove);
    Set<ConnectionDTO> connectionsWithQueue = new HashSet<>();
    // first validate the queues are empty ... if not warn user the following ports cant be deleted and rollback
    connectionsToRemove.stream().forEach(connection -> {
        Optional<ConnectionStatusEntity> connectionStatus = nifiRestClient.getNiFiRestClient().connections().getConnectionStatus(connection.getId());
        if (connectionStatus.isPresent() && connectionStatus.get().getConnectionStatus().getAggregateSnapshot().getFlowFilesQueued() > 0) {
            connectionsWithQueue.add(connection);
        }
    });
    if (!connectionsWithQueue.isEmpty()) {
        removalData.addErrorMessage("Unable to remove inputPort and connection for :" + connectionsWithQueue.stream().map(c -> c.getSource().getName()).collect(Collectors.joining(",")) + ". The Queues are not empty." + (removalData.getTemplateName() != null ? "Failure for template: " + removalData.getTemplateName() : ""), false);
        return false;
    } else {
        if (!remoteInputPortIdsToRemove.isEmpty()) {
            // verify we are allowed to delete this input port
            // if there are any remoteprocessgroups connected to this input port then we should not be allowed to delete it
            ProcessGroupStatusDTO flow = nifiRestClient.getNiFiRestClient().processGroups().flowStatus("root", true);
            boolean remoteProcessGroupsExist = flow.getAggregateSnapshot().getProcessGroupStatusSnapshots().stream().flatMap(s -> RemoteProcessGroupStatusHelper.flattened(s.getProcessGroupStatusSnapshot())).map(remoteProcessGroupStatusSnapshotEntity -> nifiRestClient.getNiFiRestClient().remoteProcessGroups().findById(remoteProcessGroupStatusSnapshotEntity.getId())).filter(Optional::isPresent).map(Optional::get).flatMap(remoteProcessGroupDTO -> remoteProcessGroupDTO.getContents().getInputPorts().stream()).anyMatch(remoteProcessGroupPortDTO -> remoteInputPortIdsToRemove.contains(remoteProcessGroupPortDTO.getId()));
            if (remoteProcessGroupsExist) {
                String msg = "Unable to remove inputPort  There are feed flows with RemotProcessGroups that are linked to one or more of the input ports you are trying to remove" + (removalData.getTemplateName() != null ? ", for template: " + removalData.getTemplateName() : "");
                removalData.addErrorMessage(msg, true);
                // remoteProcessGroupOption.getErrorMessages().add(msg);
                return false;
            } else {
                connectionsToRemove.stream().forEach(connection -> {
                    nifiRestClient.deleteConnection(connection, false);
                    removalData.addDeletedConnection(connection);
                    try {
                        PortDTO deletedPort = nifiRestClient.getNiFiRestClient().ports().deleteInputPort(connection.getSource().getId());
                        if (deletedPort != null) {
                            removalData.addDeletedPort(deletedPort);
                        }
                    } catch (NifiComponentNotFoundException e) {
                    // this is ok to catch as its deleted already
                    }
                });
            }
        }
        String successMsg = "Removed inputPort and connection for :" + connectionsToRemove.stream().map(c -> c.getSource().getName()).collect(Collectors.joining(",")) + " for template: " + (removalData.getTemplateName() != null ? removalData.getTemplateName() : "");
    }
    return true;
}

Example 37 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class DefaultTemplateExporter method gatherConnectedReusableTemplates.

private void gatherConnectedReusableTemplates(List<String> connectingReusableTemplates, Set<String> connectedTemplateIds, Set<ReusableTemplateConnectionInfo> connectingOutputPortConnectionMetadata, List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos, ProcessGroupFlowDTO reusableTemplateFlow) {
    for (ReusableTemplateConnectionInfo reusableTemplateConnectionInfo : reusableTemplateConnectionInfos) {
        String inputName = reusableTemplateConnectionInfo.getReusableTemplateInputPortName();
        // find the process group instance in the 'reusable_templates' group in NiFi for this input port
        Optional<ProcessGroupDTO> processGroupDTO = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getName().equals(inputName) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).flatMap(connectionDTO -> reusableTemplateFlow.getFlow().getProcessGroups().stream().map(processGroupEntity -> processGroupEntity.getComponent()).filter(groupDTO -> groupDTO.getId().equals(connectionDTO.getDestination().getGroupId()))).findFirst();
        if (processGroupDTO.isPresent()) {
            // walk the output ports to find any other connecting templates
            List<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new ArrayList<>();
            List<ConnectionDTO> outputPortConnections = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getGroupId().equals(processGroupDTO.get().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)).collect(Collectors.toList());
            for (ConnectionDTO outputConnection : outputPortConnections) {
                // walk these and get their templates
                // first get the connection metadata info needed
                // 1 find the reusable template input port for this connected template
                Optional<ConnectionDTO> inputPortToProcessGroupConnection = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getDestination().getId().equals(outputConnection.getDestination().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).findFirst();
                if (inputPortToProcessGroupConnection.isPresent()) {
                    ReusableTemplateConnectionInfo connectionInfo = new ReusableTemplateConnectionInfo();
                    connectionInfo.setFeedOutputPortName(outputConnection.getSource().getName());
                    connectionInfo.setReusableTemplateInputPortName(inputPortToProcessGroupConnection.get().getSource().getName());
                    connectionInfo.setInputPortDisplayName(inputPortToProcessGroupConnection.get().getSource().getName());
                    String processGroupName = reusableTemplateFlow.getFlow().getProcessGroups().stream().filter(processGroupEntity -> processGroupEntity.getComponent().getId().equals(inputPortToProcessGroupConnection.get().getDestination().getGroupId())).map(processGroupEntity -> processGroupEntity.getComponent().getName()).findFirst().orElse(null);
                    connectionInfo.setReusableTemplateProcessGroupName(processGroupName);
                    outputPortConnectionMetadata.add(connectionInfo);
                    // recursively walk these flows and gather other output port connections
                    connectingOutputPortConnectionMetadata.add(connectionInfo);
                }
            // also add in the process group if it doesnt connect
            }
            if (!outputPortConnectionMetadata.isEmpty()) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, connectingOutputPortConnectionMetadata, outputPortConnectionMetadata, reusableTemplateFlow);
            }
        }
        // find the template that has the input port name
        Map<String, String> map = nifiRestClient.getTemplatesAsXmlMatchingInputPortName(inputName, reusableTemplateConnectionInfo.getReusableTemplateProcessGroupName());
        if (map != null && !map.isEmpty()) {
            for (Map.Entry<String, String> entry : map.entrySet()) {
                String portTemplateId = entry.getKey();
                if (!connectedTemplateIds.contains(portTemplateId)) {
                    connectedTemplateIds.add(portTemplateId);
                    connectingReusableTemplates.add(entry.getValue());
                }
            }
        }
    }
}

Example 38 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getReusableTemplateProcessorsForInputPorts.

/**
 * Return all the processors that are connected to a given NiFi input port
 *
 * @param inputPortIds the ports to inspect
 * @return all the processors that are connected to a given NiFi input port
 */
public List<RegisteredTemplate.Processor> getReusableTemplateProcessorsForInputPorts(List<String> inputPortIds) {
    Set<ProcessorDTO> processorDTOs = new HashSet<>();
    if (inputPortIds != null && !inputPortIds.isEmpty()) {
        ProcessGroupDTO processGroup = nifiRestClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
        if (processGroup != null) {
            // fetch the Content
            ProcessGroupDTO content = nifiRestClient.getProcessGroup(processGroup.getId(), true, true);
            processGroup.setContents(content.getContents());
            Set<PortDTOWithGroupInfo> ports = getReusableFeedInputPorts();
            ports.stream().filter(portDTO -> inputPortIds.contains(portDTO.getId())).forEach(port -> {
                List<ConnectionDTO> connectionDTOs = NifiConnectionUtil.findConnectionsMatchingSourceId(processGroup.getContents().getConnections(), port.getId());
                if (connectionDTOs != null) {
                    connectionDTOs.stream().forEach(connectionDTO -> {
                        String processGroupId = connectionDTO.getDestination().getGroupId();
                        Set<ProcessorDTO> processors = nifiRestClient.getProcessorsForFlow(processGroupId);
                        if (processors != null) {
                            processorDTOs.addAll(processors);
                        }
                    });
                }
            });
        }
    }
    List<RegisteredTemplate.Processor> processorProperties = processorDTOs.stream().map(processorDTO -> registeredTemplateUtil.toRegisteredTemplateProcessor(processorDTO, true)).collect(Collectors.toList());
    return processorProperties;
}

Example 39 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getNiFiTemplateFlowProcessors.

/**
 * For a given Template and its related connection info to the reusable templates, walk the graph to return the Processors.
 * The system will first walk the incoming templateid.  If the {@code connectionInfo} parameter is set it will make the connections to the incoming template and continue walking those processors
 *
 * @param nifiTemplateId the NiFi templateId required to start walking the flow
 * @param connectionInfo the connections required to connect
 * @return a list of all the processors for a template and possible connections
 */
public List<RegisteredTemplate.FlowProcessor> getNiFiTemplateFlowProcessors(String nifiTemplateId, List<ReusableTemplateConnectionInfo> connectionInfo) {
    TemplateDTO templateDTO = nifiRestClient.getTemplateById(nifiTemplateId);
    // make the connection
    if (connectionInfo != null && !connectionInfo.isEmpty()) {
        Set<PortDTO> templatePorts = templateDTO.getSnippet().getOutputPorts();
        Map<String, PortDTO> outputPorts = templateDTO.getSnippet().getOutputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        Map<String, PortDTO> inputPorts = getReusableFeedInputPorts().stream().collect(Collectors.toMap(portDTO -> portDTO.getName(), Function.identity()));
        connectionInfo.stream().forEach(reusableTemplateConnectionInfo -> {
            PortDTO outputPort = outputPorts.get(reusableTemplateConnectionInfo.getFeedOutputPortName());
            PortDTO inputPort = inputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName());
            ConnectionDTO connectionDTO = new ConnectionDTO();
            ConnectableDTO source = new ConnectableDTO();
            source.setName(reusableTemplateConnectionInfo.getFeedOutputPortName());
            source.setType(outputPort.getType());
            source.setId(outputPort.getId());
            source.setGroupId(outputPort.getParentGroupId());
            ConnectableDTO dest = new ConnectableDTO();
            dest.setName(inputPort.getName());
            dest.setType(inputPort.getType());
            dest.setId(inputPort.getId());
            dest.setGroupId(inputPort.getParentGroupId());
            connectionDTO.setSource(source);
            connectionDTO.setDestination(dest);
            connectionDTO.setId(UUID.randomUUID().toString());
            templateDTO.getSnippet().getConnections().add(connectionDTO);
        });
    }
    NifiFlowProcessGroup template = nifiRestClient.getTemplateFeedFlow(templateDTO);
    return template.getProcessorMap().values().stream().map(flowProcessor -> {
        RegisteredTemplate.FlowProcessor p = new RegisteredTemplate.FlowProcessor(flowProcessor.getId());
        p.setGroupId(flowProcessor.getParentGroupId());
        p.setType(flowProcessor.getType());
        p.setName(flowProcessor.getName());
        p.setFlowId(flowProcessor.getFlowId());
        p.setIsLeaf(flowProcessor.isLeaf());
        return p;
    }).collect(Collectors.toList());
}

Example 40 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project nifi by apache.

the class StandardNiFiServiceFacade method createConnection.

@Override
public ConnectionEntity createConnection(final Revision revision, final String groupId, final ConnectionDTO connectionDTO) {
    final RevisionUpdate<ConnectionDTO> snapshot = createComponent(revision, connectionDTO, () -> connectionDAO.createConnection(groupId, connectionDTO), connection -> dtoFactory.createConnectionDto(connection));
    final Connection connection = connectionDAO.getConnection(connectionDTO.getId());
    final PermissionsDTO permissions = dtoFactory.createPermissionsDto(connection);
    final ConnectionStatusDTO status = dtoFactory.createConnectionStatusDto(controllerFacade.getConnectionStatus(connectionDTO.getId()));
    return entityFactory.createConnectionEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, status);
}