Examples with ConnectionDTO org.apache.nifi.web.api.dto.ConnectionDTO used on opensource projects

Example 26 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class ImportReusableTemplate method validateOutputPortConnections.

private boolean validateOutputPortConnections(NifiProcessGroup newTemplateInstance) {
    // Validate port connections
    newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().forEach(portDTO -> {
        if (portDTO.getValidationErrors() != null && !portDTO.getValidationErrors().isEmpty()) {
            importTemplate.setReusableFlowOutputPortConnectionsNeeded(true);
        }
        ReusableTemplateConnectionInfo connectionInfo = new ReusableTemplateConnectionInfo();
        connectionInfo.setFeedOutputPortName(portDTO.getName());
        // attempt to prefill it with the previous connection if it existed
        ConnectionDTO reusableTemplateInputPortConnection = findReusableTemplateInputPortConnectionForOutputPort(portDTO);
        if (reusableTemplateInputPortConnection != null) {
            connectionInfo.setInputPortDisplayName(reusableTemplateInputPortConnection.getSource().getName());
            connectionInfo.setReusableTemplateInputPortName(reusableTemplateInputPortConnection.getSource().getName());
            String processGroupName = findReusableTemplateProcessGroup(reusableTemplateInputPortConnection.getDestination().getGroupId()).map(processGroupDTO -> processGroupDTO.getName()).orElse(null);
            connectionInfo.setReusableTemplateProcessGroupName(processGroupName);
        }
        importTemplate.addReusableTemplateConnection(connectionInfo);
    });
    return importTemplate.isSuccess() && importTemplate.isValid() && !importTemplate.isReusableFlowOutputPortConnectionsNeeded();
}

Example 27 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class ImportReusableTemplate method recreateOutputPortConnections.

private void recreateOutputPortConnections(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    VersionedProcessGroup versionedProcessGroup = newTemplateInstance.getVersionedProcessGroup();
    String templateName = importTemplate.getTemplateName();
    // Dest == input port in versioned off process group
    if (versionedProcessGroup != null) {
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ConnectionDTO connectionDTO : versionedProcessGroup.getDeletedInputPortConnections()) {
                if (connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)) {
                    // connect
                    PortDTO destPort = newTemplateInstance.getProcessGroupEntity().getContents().getInputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionDTO.getDestination().getName()) && connectionDTO.getDestination().getGroupId().equalsIgnoreCase(newTemplateInstance.getVersionedProcessGroup().getProcessGroupPriorToVersioning().getId())).findFirst().orElse(null);
                    if (destPort != null) {
                        // make the connection now from the output port to the 'connectionToUse' destination
                        ConnectableDTO source = NifiConnectionUtil.asNewConnectable(connectionDTO.getSource());
                        ConnectableDTO dest = NifiConnectionUtil.asConnectable(destPort);
                        ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                        connections.add(newConnection);
                        // possibly store the ports too?
                        log.info("Reconnected output port {} ({}) to this new process group input port:  {} {{}) ", source.getName(), source.getId(), dest.getName(), dest.getId());
                    } else {
                        // ERROR cant recreate previous connections that were going into this reusable template
                        String msg = "Unable to recreate the connection for template: " + templateName + " that was previously connected to this template prior to the update. The following connection is missing:  Connecting ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "'].";
                        log.error(msg);
                        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, msg, "");
                        importStatusMessage.update("Unable to establish prior connection for reusable template: " + templateName + ".  Connection:  ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "']", false);
                        break;
                    }
                }
            }
        }
    }
}

Example 28 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class TemplateCreationHelper method versionProcessGroup.

/**
 * Version a ProcessGroup renaming it with the name - {timestamp millis}.
 * If {@code removeIfInactive} is true it will not version but just delete it
 *
 * @param processGroup the group to version
 */
public VersionedProcessGroup versionProcessGroup(ProcessGroupDTO processGroup, String versionIdentifier) {
    log.info("Versioning Process Group {} ", processGroup.getName());
    VersionedProcessGroup versionedProcessGroup = new VersionedProcessGroup();
    versionedProcessGroup.setProcessGroupPriorToVersioning(processGroup);
    versionedProcessGroup.setProcessGroupName(processGroup.getName());
    List<ProcessorDTO> inputProcessorsPriorToDisabling = restClient.disableAllInputProcessors(processGroup.getId());
    versionedProcessGroup.setInputProcessorsPriorToDisabling(inputProcessorsPriorToDisabling);
    log.info("Disabled Inputs for {} ", processGroup.getName());
    // attempt to stop all processors
    try {
        restClient.stopInputs(processGroup.getId());
        log.info("Stopped Input Ports for {}, ", processGroup.getName());
    } catch (Exception e) {
        log.error("Error trying to stop Input Ports for {} while creating a new version ", processGroup.getName());
    }
    // delete input connections
    try {
        List<ConnectionDTO> deletedConnections = deleteInputPortConnections(processGroup);
        versionedProcessGroup.setDeletedInputPortConnections(deletedConnections);
    } catch (NifiClientRuntimeException e) {
        log.error("Error trying to delete input port connections for Process Group {} while creating a new version. ", processGroup.getName(), e);
        getErrors().add(new NifiError(NifiError.SEVERITY.FATAL, "The input port connections to the process group " + processGroup.getName() + " could not be deleted. Please delete them manually " + "in NiFi and try again."));
    }
    String versionedProcessGroupName = getVersionedProcessGroupName(processGroup.getName(), versionIdentifier);
    versionedProcessGroup.setVersionedProcessGroupName(versionedProcessGroupName);
    // rename the feedGroup to be name+timestamp
    processGroup.setName(versionedProcessGroupName);
    restClient.updateProcessGroup(processGroup);
    log.info("Renamed ProcessGroup to  {}, ", processGroup.getName());
    versionedProcessGroup.setVersionedProcessGroup(processGroup);
    return versionedProcessGroup;
}

Example 29 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class TemplateCreationHelper method deleteInputPortConnections.

/**
 * Deletes the input port connections to the specified process group.
 *
 * <p>When versioning we want to delete only the input port connections. Keep output port connections in place as they may still have data running through them that should flow through the
 * system.</p>
 *
 * @param processGroup the process group with input port connections
 * @throws NifiClientRuntimeException if a connection cannot be deleted
 */
private List<ConnectionDTO> deleteInputPortConnections(@Nonnull final ProcessGroupDTO processGroup) throws NifiClientRuntimeException {
    // Get the list of incoming connections coming from some source to this process group
    List<ConnectionDTO> deletedConnections = new ArrayList<>();
    final Set<ConnectionDTO> connectionsEntity = nifiObjectCache.isCacheConnections() ? nifiObjectCache.getConnections(processGroup.getParentGroupId()) : restClient.getProcessGroupConnections(processGroup.getParentGroupId());
    if (connectionsEntity == null) {
        return deletedConnections;
    }
    final List<ConnectionDTO> connections = NifiConnectionUtil.findConnectionsMatchingDestinationGroupId(connectionsEntity, processGroup.getId());
    if (connections == null) {
        return deletedConnections;
    }
    Set<String> removedConnections = new HashSet<>();
    // Delete the connections
    for (ConnectionDTO connection : connections) {
        final String type = connection.getSource().getType();
        log.info("Found connection {} matching source type {} and destination group {}.", connection.getId(), type, connection.getDestination().getId());
        // Stop the port
        if (NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name().equalsIgnoreCase(type)) {
            try {
                restClient.stopInputPort(connection.getSource().getGroupId(), connection.getSource().getId());
                log.info("Stopped input port {} for connection: {} ", connection.getSource().getId(), connection.getId());
            } catch (Exception e) {
                log.error("Failed to stop input port for connection: {}", connection.getId(), e);
                throw new NifiClientRuntimeException("Error stopping the input port " + connection.getSource().getId() + " for connection " + connection.getId() + " prior to deleting the " + "connection.");
            }
        }
        // Delete the connection
        try {
            restClient.deleteConnection(connection, false);
            removedConnections.add(connection.getId());
            deletedConnections.add(connection);
        } catch (Exception e) {
            log.error("Failed to delete the connection: {}", connection.getId(), e);
            final String source = (connection.getSource() != null) ? connection.getSource().getName() : null;
            final String destination = (connection.getDestination() != null) ? connection.getDestination().getName() : null;
            throw new NifiClientRuntimeException("Error deleting the connection " + connection.getId() + " with source " + source + " and destination " + destination + ".");
        }
    }
    nifiObjectCache.removeConnections(processGroup.getParentGroupId(), removedConnections);
    return deletedConnections;
}

Example 30 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class RemoteProcessGroupValidator method updateRemoteConnection.

/**
 * When updating the connections to remote process groups the system needs to detect if the RemoteProcessGroup is valid (has no issues connecting to the targetURI prior to making the connection)
 * This call will try and sleep and retry if the remoteProcessGroup has connection issues
 *
 * @param remoteProcessGroupConnectionDTO the connection to update
 * @param validation                      the holder of validation success/failures
 * @param retryCount                      the number of retries already attempted for this connection
 * @return true if successful, false if not.  The Validation object will also be populated with the validation information
 */
private boolean updateRemoteConnection(UpdatedConnection updatedConnection, RemoteProcessGroupValidation validation, int retryCount) {
    // ensure we are not attempting to authorize
    // TODO pull timeouts to configurable parameters
    boolean success = false;
    final int sleepTimeMillis = templateConnectionUtil.getRemoteProcessGroupSleepTime();
    final int maxAttempts = templateConnectionUtil.getRemoteProcessGroupMaxAttempts();
    ConnectionDTO connectionDTO = updatedConnection.getNewConnection();
    // set the destination to that id
    if (updatedConnection.getRemoteProcessGroupDTO().hasRemoteProcessGroupAuthorizationIssues() || updatedConnection.getRemoteProcessGroupDTO().getRemoteProcessGroup().getInputPortCount() == 0) {
        // wait
        log.info("Authorization issue found when attempting to update Remote Process Group Port connection for {}.  Retry Attempt: {} ", connectionDTO.getDestination().getName(), retryCount);
        if (retryCount <= maxAttempts) {
            Uninterruptibles.sleepUninterruptibly(sleepTimeMillis, TimeUnit.MILLISECONDS);
            Optional<RemoteProcessGroupDTO> remoteProcessGroupDTO = restClient.getNiFiRestClient().remoteProcessGroups().findById(updatedConnection.getRemoteProcessGroupDTO().getRemoteProcessGroup().getId());
            if (remoteProcessGroupDTO.isPresent()) {
                updatedConnection.updateRemoteProcessGroup(remoteProcessGroupDTO.get());
                retryCount++;
                return updateRemoteConnection(updatedConnection, validation, retryCount);
            } else {
                validation.addInvalidConnection(connectionDTO);
                success = false;
            }
        } else {
            validation.addInvalidConnection(connectionDTO);
            success = false;
        }
    } else if (connectionDTO.getId() == null) {
        try {
            ConnectionDTO connection = new ConnectionDTO();
            connection.setSource(connectionDTO.getSource());
            connection.setDestination(connectionDTO.getDestination());
            connection.setParentGroupId(connectionDTO.getParentGroupId());
            connection.setSelectedRelationships(new HashSet<>());
            connection.getSelectedRelationships().add("success");
            connection.setName(StringUtils.isNotBlank(connectionDTO.getName()) ? connectionDTO.getName() : (connectionDTO.getSource().getName() + " - " + connectionDTO.getDestination().getName()));
            connection = restClient.getNiFiRestClient().processGroups().createConnection(connection);
            if (connection != null) {
                updatedConnection.setNewConnection(connection);
                updatedConnection.setUpdated(true);
                success = true;
            } else {
                success = false;
            }
        } catch (Exception e) {
            log.info("Error found attempting to create the new connection to the Remote Process Group for {} in Parent Process Group: {}.  Retry Attempt: {} ", connectionDTO.getDestination().getName(), connectionDTO.getParentGroupId(), retryCount);
            if (retryCount <= maxAttempts) {
                Uninterruptibles.sleepUninterruptibly(sleepTimeMillis, TimeUnit.MILLISECONDS);
                retryCount++;
                return updateRemoteConnection(updatedConnection, validation, retryCount);
            } else {
                // unable to create new connection
                validation.addInvalidConnection(connectionDTO);
                success = false;
            }
        }
    } else {
        log.info("Updating Remote Process Group Port connection for {} ", connectionDTO.getDestination().getName());
        try {
            Optional<ConnectionDTO> connection = restClient.getNiFiRestClient().connections().update(connectionDTO);
            if (connection.isPresent()) {
                validation.addUpdatedConnection(connection.get());
                success = true;
            } else {
                validation.addInvalidConnection(connectionDTO);
                success = false;
            }
        } catch (Exception e) {
            validation.addInvalidConnection(connectionDTO);
            success = false;
        }
    }
    return success;
}