Examples with ConnectableDTO org.apache.nifi.web.api.dto.ConnectableDTO used on opensource projects

Example 21 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class TemplateConnectionUtil method connectFeedToReusableTemplate.

public void connectFeedToReusableTemplate(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup, List<InputOutputPort> inputOutputPorts) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            if (categoryOutputPort != null) {
                // ensure it exists
                try {
                    categoryOutputPort = restClient.getNiFiRestClient().ports().getOutputPort(categoryOutputPort.getId());
                } catch (Exception e) {
                    categoryOutputPort = null;
                }
            }
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    // ensure the port exists
                    niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 22 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class ImportReusableTemplate method validateInstance.

public boolean validateInstance() {
    boolean valid = importTemplate.getTemplateResults().isSuccess() && importTemplate.isValid() && !importTemplate.isReusableFlowOutputPortConnectionsNeeded();
    importTemplate.setSuccess(valid);
    if (importTemplate.isReusableFlowOutputPortConnectionsNeeded()) {
        uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Additional Port Connection information is necessary for the template " + importTemplate.getTemplateName(), true, false);
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE);
    if (valid) {
        // start all the connections we created
        List<ConnectionDTO> createdConnections = getConnections();
        if (!createdConnections.isEmpty()) {
            UploadProgressMessage message = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Verify and start all input/output port connections for the template " + importTemplate.getTemplateName());
            getConnections().stream().forEach(connectionDTO -> {
                // verify the Source is started
                ConnectableDTO connectableDTO = connectionDTO.getSource();
                boolean started = startConnectablePorts(connectableDTO);
                // verify the Destination is started
                connectableDTO = connectionDTO.getDestination();
                started &= startConnectablePorts(connectableDTO);
                if (!started) {
                    importTemplate.setSuccess(false);
                }
            });
            message.complete(importTemplate.isSuccess());
        }
    }
    return valid;
}

Example 23 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project kylo by Teradata.

the class ImportReusableTemplate method connectReusableFlow.

private ProcessGroupDTO connectReusableFlow(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    String templateName = this.importTemplate.getTemplateName();
    ImportComponentOption componentOption = importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION);
    ProcessGroupDTO processGroupDTO = newTemplateInstance.getProcessGroupEntity();
    Set<PortDTO> inputPorts = getReusableTemplateInputPorts();
    Set<ConnectionDTO> reusableTemplateConnections = getReusableTemplateConnections();
    if (componentOption != null && !componentOption.getConnectionInfo().isEmpty()) {
        // connect the output port to the input port
        // we are connecting a reusable template back to another reusable template
        // follow the input port destination connection to its internal process group port
        Set<ConnectionDTO> newConnections = new HashSet<>();
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ReusableTemplateConnectionInfo connectionInfo : componentOption.getConnectionInfo()) {
                String reusableTemplateInputPortName = connectionInfo.getReusableTemplateInputPortName();
                // find the portdto matching this name in the reusable template group
                /**
                 * The connection coming from the 'reusableTemplateInputPortName' to the next input port
                 */
                Optional<ConnectionDTO> connectionToUse = Optional.empty();
                /**
                 * The port that matches the 'reusableTemplateInputPortName connection destination
                 */
                Optional<PortDTO> sourcePort = Optional.empty();
                connectionToUse = inputPorts.stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(reusableTemplateInputPortName)).findFirst().flatMap(portToInspect -> reusableTemplateConnections.stream().filter(connectionDTO -> connectionDTO.getDestination().getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name()) && connectionDTO.getSource().getId().equalsIgnoreCase(portToInspect.getId())).findFirst());
                if (connectionToUse.isPresent()) {
                    sourcePort = newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionInfo.getFeedOutputPortName())).findFirst();
                }
                if (sourcePort.isPresent()) {
                    // make the connection now from the output port to the 'connectionToUse' destination
                    ConnectableDTO source = NifiConnectionUtil.asConnectable(sourcePort.get());
                    ConnectableDTO dest = NifiConnectionUtil.asNewConnectable(connectionToUse.get().getDestination());
                    ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                    newConnections.add(newConnection);
                    connections.add(newConnection);
                    log.info("Connected the output port {} ({}) to another reusable template input port: {} {{}).  The public reusable template port name is: {} ", source.getName(), source.getId(), dest.getName(), dest.getId(), reusableTemplateInputPortName);
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connected this template '" + templateName + "' with '" + reusableTemplateInputPortName + "' connected to '" + sourcePort.get().getName() + "'", true, true);
                } else {
                // log.error("Unable to find a connection to connect the reusable template together.  Please verify the Input Port named '{}' under the 'reusable_templates' group has a connection going to another input port",reusableTemplateInputPortName);
                // importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, "Unable to connect the reusable template to the designated input port: '" + reusableTemplateInputPortName + "'. Please verify the Input Port named '" + reusableTemplateInputPortName + "' under the 'reusable_templates' group has a connection going to another input port.  You may need to re-import the template with this input port. ", "");
                // importStatusMessage.update("Unable to connect the reusable template to the designated input port: "+reusableTemplateInputPortName, false);
                // uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to connect this template '"+templateName+"' with '"+reusableTemplateInputPortName,true,false);
                // break;
                }
            }
        }
        if (!newConnections.isEmpty()) {
            // requery for the ports to check validity again
            processGroupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(newTemplateInstance.getProcessGroupEntity().getId(), false, true).orElse(processGroupDTO);
            // reset it back to the newTemplateInstance
            newTemplateInstance.updateProcessGroupContent(processGroupDTO);
        }
    }
    return processGroupDTO;
}

Example 24 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project nifi by apache.

the class SnippetAuditor method auditSnippet.

/**
 * Audits the specified snippet.
 */
private void auditSnippet(final FlowSnippetDTO snippet) {
    final Collection<Action> actions = new ArrayList<>();
    final Date timestamp = new Date();
    // input ports
    for (final PortDTO inputPort : snippet.getInputPorts()) {
        actions.add(generateAuditRecord(inputPort.getId(), inputPort.getName(), Component.InputPort, Operation.Add, timestamp));
    }
    // output ports
    for (final PortDTO outputPort : snippet.getOutputPorts()) {
        actions.add(generateAuditRecord(outputPort.getId(), outputPort.getName(), Component.OutputPort, Operation.Add, timestamp));
    }
    // remote processor groups
    for (final RemoteProcessGroupDTO remoteProcessGroup : snippet.getRemoteProcessGroups()) {
        FlowChangeRemoteProcessGroupDetails remoteProcessGroupDetails = new FlowChangeRemoteProcessGroupDetails();
        remoteProcessGroupDetails.setUri(remoteProcessGroup.getTargetUri());
        final FlowChangeAction action = generateAuditRecord(remoteProcessGroup.getId(), remoteProcessGroup.getName(), Component.RemoteProcessGroup, Operation.Add, timestamp);
        action.setComponentDetails(remoteProcessGroupDetails);
        actions.add(action);
    }
    // processor groups
    for (final ProcessGroupDTO processGroup : snippet.getProcessGroups()) {
        actions.add(generateAuditRecord(processGroup.getId(), processGroup.getName(), Component.ProcessGroup, Operation.Add, timestamp));
    }
    // processors
    for (final ProcessorDTO processor : snippet.getProcessors()) {
        final FlowChangeExtensionDetails processorDetails = new FlowChangeExtensionDetails();
        processorDetails.setType(StringUtils.substringAfterLast(processor.getType(), "."));
        final FlowChangeAction action = generateAuditRecord(processor.getId(), processor.getName(), Component.Processor, Operation.Add, timestamp);
        action.setComponentDetails(processorDetails);
        actions.add(action);
    }
    // funnels
    for (final FunnelDTO funnel : snippet.getFunnels()) {
        actions.add(generateAuditRecord(funnel.getId(), StringUtils.EMPTY, Component.Funnel, Operation.Add, timestamp));
    }
    // connections
    for (final ConnectionDTO connection : snippet.getConnections()) {
        final ConnectableDTO source = connection.getSource();
        final ConnectableDTO destination = connection.getDestination();
        // determine the relationships and connection name
        final String relationships = CollectionUtils.isEmpty(connection.getSelectedRelationships()) ? StringUtils.EMPTY : StringUtils.join(connection.getSelectedRelationships(), ", ");
        final String name = StringUtils.isBlank(connection.getName()) ? relationships : connection.getName();
        // create the connect details
        FlowChangeConnectDetails connectDetails = new FlowChangeConnectDetails();
        connectDetails.setSourceId(source.getId());
        connectDetails.setSourceName(source.getName());
        connectDetails.setSourceType(determineConnectableType(source));
        connectDetails.setRelationship(relationships);
        connectDetails.setDestinationId(destination.getId());
        connectDetails.setDestinationName(destination.getName());
        connectDetails.setDestinationType(determineConnectableType(destination));
        // create the audit record
        final FlowChangeAction action = generateAuditRecord(connection.getId(), name, Component.Connection, Operation.Connect, timestamp);
        action.setActionDetails(connectDetails);
        actions.add(action);
    }
    // save the actions
    if (!actions.isEmpty()) {
        saveActions(actions, logger);
    }
}

Example 25 with ConnectableDTO

use of org.apache.nifi.web.api.dto.ConnectableDTO in project nifi by apache.

the class StandardConnectionDAO method verifyCreate.

@Override
public void verifyCreate(String groupId, ConnectionDTO connectionDTO) {
    // validate the incoming request
    final List<String> validationErrors = validateProposedConfiguration(groupId, connectionDTO);
    // ensure there was no validation errors
    if (!validationErrors.isEmpty()) {
        throw new ValidationException(validationErrors);
    }
    // Ensure that both the source and the destination for the connection exist.
    // In the case that the source or destination is a port in a Remote Process Group,
    // this is necessary because the ports can change in the background. It may still be
    // possible for a port to disappear between the 'verify' stage and the creation stage,
    // but this prevents the case where some nodes already know about the port while other
    // nodes in the cluster do not. This is a more common case, as users may try to connect
    // to the port as soon as the port is created.
    final ConnectableDTO sourceDto = connectionDTO.getSource();
    if (sourceDto == null || sourceDto.getId() == null) {
        throw new IllegalArgumentException("Cannot create connection without specifying source");
    }
    final ConnectableDTO destinationDto = connectionDTO.getDestination();
    if (destinationDto == null || destinationDto.getId() == null) {
        throw new IllegalArgumentException("Cannot create connection without specifying destination");
    }
    if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDto.getType())) {
        final ProcessGroup sourceParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = sourceParentGroup.getRemoteProcessGroup(sourceDto.getGroupId());
        if (remoteProcessGroup == null) {
            throw new IllegalArgumentException("Unable to find the specified remote process group.");
        }
        final RemoteGroupPort sourceConnectable = remoteProcessGroup.getOutputPort(sourceDto.getId());
        if (sourceConnectable == null) {
            throw new IllegalArgumentException("The specified source for the connection does not exist");
        } else if (!sourceConnectable.getTargetExists()) {
            throw new IllegalArgumentException("The specified remote output port does not exist.");
        }
    } else {
        final ProcessGroup sourceGroup = locateProcessGroup(flowController, sourceDto.getGroupId());
        final Connectable sourceConnectable = sourceGroup.getConnectable(sourceDto.getId());
        if (sourceConnectable == null) {
            throw new IllegalArgumentException("The specified source for the connection does not exist");
        }
    }
    if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDto.getType())) {
        final ProcessGroup destinationParentGroup = locateProcessGroup(flowController, groupId);
        final RemoteProcessGroup remoteProcessGroup = destinationParentGroup.getRemoteProcessGroup(destinationDto.getGroupId());
        if (remoteProcessGroup == null) {
            throw new IllegalArgumentException("Unable to find the specified remote process group.");
        }
        final RemoteGroupPort destinationConnectable = remoteProcessGroup.getInputPort(destinationDto.getId());
        if (destinationConnectable == null) {
            throw new IllegalArgumentException("The specified destination for the connection does not exist");
        } else if (!destinationConnectable.getTargetExists()) {
            throw new IllegalArgumentException("The specified remote input port does not exist.");
        }
    } else {
        final ProcessGroup destinationGroup = locateProcessGroup(flowController, destinationDto.getGroupId());
        final Connectable destinationConnectable = destinationGroup.getConnectable(destinationDto.getId());
        if (destinationConnectable == null) {
            throw new IllegalArgumentException("The specified destination for the connection does not exist");
        }
    }
}