Examples with ConnectionDTO org.apache.nifi.web.api.dto.ConnectionDTO used on opensource projects

Example 46 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class TemplateConnectionUtil method connectFeedToReusableTemplate.

public void connectFeedToReusableTemplate(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup, List<InputOutputPort> inputOutputPorts) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            if (categoryOutputPort != null) {
                // ensure it exists
                try {
                    categoryOutputPort = restClient.getNiFiRestClient().ports().getOutputPort(categoryOutputPort.getId());
                } catch (Exception e) {
                    categoryOutputPort = null;
                }
            }
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    // ensure the port exists
                    niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 47 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class NifiFlowCacheImpl method applyClusterUpdates.

/**
 * if Kylo is clustered it needs to sync any updates from the other Kylo instances before proceeding
 */
public synchronized void applyClusterUpdates() {
    List<NifiFlowCacheClusterUpdateMessage> updates = nifiFlowCacheClusterManager.findUpdates();
    Set<String> templateUpdates = new HashSet<>();
    boolean needsUpdates = !updates.isEmpty();
    if (needsUpdates) {
        log.info("Kylo Cluster Update: Detected changes.  About to apply {} updates ", updates.size());
    }
    updates.stream().forEach(update -> {
        switch(update.getType()) {
            case FEED:
                NifiFlowCacheFeedUpdate feedUpdate = nifiFlowCacheClusterManager.getFeedUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Feed Change update for {}", feedUpdate.getFeedName());
                updateFlow(feedUpdate);
                break;
            case FEED2:
                NifiFlowCacheFeedUpdate2 feedUpdate2 = nifiFlowCacheClusterManager.getFeedUpdate2(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Feed Change update for {}", feedUpdate2.getFeedName());
                updateFlow(feedUpdate2);
                break;
            case CONNECTION:
                Collection<ConnectionDTO> connectionDTOS = nifiFlowCacheClusterManager.getConnectionsUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Connection list update");
                updateConnectionMap(connectionDTOS, false);
                if (connectionDTOS != null) {
                    connectionDTOS.stream().forEach(c -> {
                        niFiObjectCache.addConnection(c.getParentGroupId(), c);
                    });
                }
                break;
            case PROCESSOR:
                Collection<ProcessorDTO> processorDTOS = nifiFlowCacheClusterManager.getProcessorsUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Processor list update");
                updateProcessorIdNames(processorDTOS, false);
                break;
            case TEMPLATE:
                if (!templateUpdates.contains(update.getMessage())) {
                    RegisteredTemplate template = nifiFlowCacheClusterManager.getTemplate(update.getMessage());
                    log.info("Kylo Cluster Update:  Applying Template update for {} ", template.getTemplateName());
                    updateRegisteredTemplate(template, false);
                    templateUpdates.add(update.getMessage());
                }
                break;
            default:
                break;
        }
    });
    if (needsUpdates) {
        nifiFlowCacheClusterManager.appliedUpdates(updates);
        lastUpdated = DateTime.now();
        log.info("Kylo Cluster Update: NiFi Flow File Cache is in sync. All {} updates have been applied to the cache. ", updates.size());
    }
}

Example 48 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class NifiFlowCacheImpl method updateConnectionMap.

private void updateConnectionMap(Collection<ConnectionDTO> connections, boolean notifyClusterMembers) {
    Map<String, NifiFlowConnection> connectionIdToConnectionMap = new HashMap<>();
    if (connections != null) {
        connections.stream().forEach(connectionDTO -> {
            NifiFlowConnection nifiFlowConnection = NiFiFlowConnectionConverter.toNiFiFlowConnection(connectionDTO);
            if (nifiFlowConnection != null) {
                connectionIdToConnectionMap.put(nifiFlowConnection.getConnectionIdentifier(), nifiFlowConnection);
            }
        });
    }
    this.connectionIdToConnectionMap.putAll(toConnectionIdMap(connectionIdToConnectionMap.values()));
    if (connections != null) {
        Map<String, String> connectionIdToNameMap = connections.stream().collect(Collectors.toMap(conn -> conn.getId(), conn -> conn.getName()));
        connectionIdCacheNameMap.putAll(connectionIdToNameMap);
    }
    if (notifyClusterMembers) {
        if (nifiFlowCacheClusterManager.isClustered()) {
            nifiFlowCacheClusterManager.updateConnections(connections);
        }
        lastUpdated = DateTime.now();
    }
}

Example 49 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class ImportReusableTemplate method validateInstance.

public boolean validateInstance() {
    boolean valid = importTemplate.getTemplateResults().isSuccess() && importTemplate.isValid() && !importTemplate.isReusableFlowOutputPortConnectionsNeeded();
    importTemplate.setSuccess(valid);
    if (importTemplate.isReusableFlowOutputPortConnectionsNeeded()) {
        uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Additional Port Connection information is necessary for the template " + importTemplate.getTemplateName(), true, false);
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE);
    if (valid) {
        // start all the connections we created
        List<ConnectionDTO> createdConnections = getConnections();
        if (!createdConnections.isEmpty()) {
            UploadProgressMessage message = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Verify and start all input/output port connections for the template " + importTemplate.getTemplateName());
            getConnections().stream().forEach(connectionDTO -> {
                // verify the Source is started
                ConnectableDTO connectableDTO = connectionDTO.getSource();
                boolean started = startConnectablePorts(connectableDTO);
                // verify the Destination is started
                connectableDTO = connectionDTO.getDestination();
                started &= startConnectablePorts(connectableDTO);
                if (!started) {
                    importTemplate.setSuccess(false);
                }
            });
            message.complete(importTemplate.isSuccess());
        }
    }
    return valid;
}

Example 50 with ConnectionDTO

use of org.apache.nifi.web.api.dto.ConnectionDTO in project kylo by Teradata.

the class ImportReusableTemplate method rollbackTemplateImportInNifi.

/**
 * Restore the previous Template back to Nifi
 */
private void rollbackTemplateImportInNifi() {
    UploadProgressMessage rollbackMessage = restoreOldTemplateXml();
    // If we are working with a reusable flow we need to recreate the old one
    if (importTemplate.getTemplateResults() != null && importTemplate.getTemplateResults().isReusableFlowInstance()) {
        UploadProgressMessage progressMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Attempting to restore old instance for: " + importTemplate.getTemplateName());
        VersionedProcessGroup versionedProcessGroup = null;
        if (importTemplate.getTemplateResults().getVersionedProcessGroup() != null) {
            versionedProcessGroup = importTemplate.getTemplateResults().getVersionedProcessGroup();
        }
        // rename the one we created to a temp name
        ProcessGroupDTO groupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(importTemplate.getTemplateResults().getProcessGroupEntity().getId(), false, false).orElse(null);
        if (groupDTO != null) {
            String tmpName = groupDTO.getName() + "- " + System.currentTimeMillis();
            groupDTO.setName(tmpName);
            nifiRestClient.getNiFiRestClient().processGroups().update(groupDTO);
            log.info("Rollback Template: {}.  Renamed template instance that was just created to a temporary name of {}.  This will get deleted later. ", importTemplate.getTemplateName(), tmpName);
        }
        if (versionedProcessGroup != null) {
            progressMessage.update("Rollback Status: Attempting to initialize and verify prior template instance for " + importTemplate.getTemplateName());
            // rename the versioned one back
            ProcessGroupDTO oldProcessGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(versionedProcessGroup.getProcessGroupPriorToVersioning().getId(), true, true).orElse(null);
            if (oldProcessGroup != null) {
                oldProcessGroup.setName(versionedProcessGroup.getProcessGroupName());
                nifiRestClient.getNiFiRestClient().processGroups().update(oldProcessGroup);
                progressMessage.update("Rollback Status: Renamed template process group " + versionedProcessGroup.getVersionedProcessGroupName() + " back to " + versionedProcessGroup.getProcessGroupName());
            }
            // add back in the connections
            List<ConnectionDTO> createdConnections = new ArrayList<>();
            List<ConnectionDTO> connections = versionedProcessGroup.getDeletedInputPortConnections();
            if (connections != null) {
                connections.stream().forEach(connectionDTO -> {
                    createdConnections.add(nifiRestClient.getNiFiRestClient().processGroups().createConnection(connectionDTO.getParentGroupId(), connectionDTO.getSource(), connectionDTO.getDestination()));
                });
                uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Rollback Status: Recreated " + createdConnections.size() + " connections ", true, true);
            }
            List<ProcessorDTO> inputs = versionedProcessGroup.getInputProcessorsPriorToDisabling();
            if (inputs != null) {
                // update the state
                progressMessage.update("Rollback Status: Marking the process group " + versionedProcessGroup.getProcessGroupName() + " as running");
            }
        }
        if (groupDTO != null) {
            progressMessage.update("Rollback Status: Removing invalid template instance process group:  " + groupDTO.getName());
            // delete the new one
            try {
                nifiRestClient.removeProcessGroup(groupDTO.getId(), groupDTO.getParentGroupId());
            } catch (Exception e) {
                log.error("Error trying to remove invalid template instance {}", groupDTO.getName(), e);
            }
            Optional<ProcessGroupDTO> deletedGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(groupDTO.getId(), false, false);
            if (deletedGroup.isPresent()) {
                progressMessage.update("Rollback Status: Failure", false);
                rollbackMessage.update("Rollback Unsuccessful!!  The invalid group " + deletedGroup.get().getName() + " still exists.  You will need to login to NiFi and verify your reusable templates are correct!", false);
            } else {
                String message = "Rollback Status: Success.";
                if (versionedProcessGroup != null) {
                    message += " Restored '" + versionedProcessGroup.getVersionedProcessGroupName() + "' back to '" + importTemplate.getTemplateName() + "'";
                }
                progressMessage.update(message, true);
                rollbackMessage.update("Rollback Successful!", true);
            }
        }
    } else {
        rollbackMessage.update("Rollback Successful!", true);
    }
}