Examples with ProcessGroupDTO org.apache.nifi.web.api.dto.ProcessGroupDTO used on opensource projects

Example 81 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class TemplateConnectionUtil method connectFeedToReusableTemplate.

public void connectFeedToReusableTemplate(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup, List<InputOutputPort> inputOutputPorts) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            if (categoryOutputPort != null) {
                // ensure it exists
                try {
                    categoryOutputPort = restClient.getNiFiRestClient().ports().getOutputPort(categoryOutputPort.getId());
                } catch (Exception e) {
                    categoryOutputPort = null;
                }
            }
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    // ensure the port exists
                    niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 82 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class ImportReusableTemplate method rollbackTemplateImportInNifi.

/**
 * Restore the previous Template back to Nifi
 */
private void rollbackTemplateImportInNifi() {
    UploadProgressMessage rollbackMessage = restoreOldTemplateXml();
    // If we are working with a reusable flow we need to recreate the old one
    if (importTemplate.getTemplateResults() != null && importTemplate.getTemplateResults().isReusableFlowInstance()) {
        UploadProgressMessage progressMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Attempting to restore old instance for: " + importTemplate.getTemplateName());
        VersionedProcessGroup versionedProcessGroup = null;
        if (importTemplate.getTemplateResults().getVersionedProcessGroup() != null) {
            versionedProcessGroup = importTemplate.getTemplateResults().getVersionedProcessGroup();
        }
        // rename the one we created to a temp name
        ProcessGroupDTO groupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(importTemplate.getTemplateResults().getProcessGroupEntity().getId(), false, false).orElse(null);
        if (groupDTO != null) {
            String tmpName = groupDTO.getName() + "- " + System.currentTimeMillis();
            groupDTO.setName(tmpName);
            nifiRestClient.getNiFiRestClient().processGroups().update(groupDTO);
            log.info("Rollback Template: {}.  Renamed template instance that was just created to a temporary name of {}.  This will get deleted later. ", importTemplate.getTemplateName(), tmpName);
        }
        if (versionedProcessGroup != null) {
            progressMessage.update("Rollback Status: Attempting to initialize and verify prior template instance for " + importTemplate.getTemplateName());
            // rename the versioned one back
            ProcessGroupDTO oldProcessGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(versionedProcessGroup.getProcessGroupPriorToVersioning().getId(), true, true).orElse(null);
            if (oldProcessGroup != null) {
                oldProcessGroup.setName(versionedProcessGroup.getProcessGroupName());
                nifiRestClient.getNiFiRestClient().processGroups().update(oldProcessGroup);
                progressMessage.update("Rollback Status: Renamed template process group " + versionedProcessGroup.getVersionedProcessGroupName() + " back to " + versionedProcessGroup.getProcessGroupName());
            }
            // add back in the connections
            List<ConnectionDTO> createdConnections = new ArrayList<>();
            List<ConnectionDTO> connections = versionedProcessGroup.getDeletedInputPortConnections();
            if (connections != null) {
                connections.stream().forEach(connectionDTO -> {
                    createdConnections.add(nifiRestClient.getNiFiRestClient().processGroups().createConnection(connectionDTO.getParentGroupId(), connectionDTO.getSource(), connectionDTO.getDestination()));
                });
                uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Rollback Status: Recreated " + createdConnections.size() + " connections ", true, true);
            }
            List<ProcessorDTO> inputs = versionedProcessGroup.getInputProcessorsPriorToDisabling();
            if (inputs != null) {
                // update the state
                progressMessage.update("Rollback Status: Marking the process group " + versionedProcessGroup.getProcessGroupName() + " as running");
            }
        }
        if (groupDTO != null) {
            progressMessage.update("Rollback Status: Removing invalid template instance process group:  " + groupDTO.getName());
            // delete the new one
            try {
                nifiRestClient.removeProcessGroup(groupDTO.getId(), groupDTO.getParentGroupId());
            } catch (Exception e) {
                log.error("Error trying to remove invalid template instance {}", groupDTO.getName(), e);
            }
            Optional<ProcessGroupDTO> deletedGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(groupDTO.getId(), false, false);
            if (deletedGroup.isPresent()) {
                progressMessage.update("Rollback Status: Failure", false);
                rollbackMessage.update("Rollback Unsuccessful!!  The invalid group " + deletedGroup.get().getName() + " still exists.  You will need to login to NiFi and verify your reusable templates are correct!", false);
            } else {
                String message = "Rollback Status: Success.";
                if (versionedProcessGroup != null) {
                    message += " Restored '" + versionedProcessGroup.getVersionedProcessGroupName() + "' back to '" + importTemplate.getTemplateName() + "'";
                }
                progressMessage.update(message, true);
                rollbackMessage.update("Rollback Successful!", true);
            }
        }
    } else {
        rollbackMessage.update("Rollback Successful!", true);
    }
}

Example 83 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class ImportReusableTemplate method connectReusableFlow.

private ProcessGroupDTO connectReusableFlow(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    String templateName = this.importTemplate.getTemplateName();
    ImportComponentOption componentOption = importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION);
    ProcessGroupDTO processGroupDTO = newTemplateInstance.getProcessGroupEntity();
    Set<PortDTO> inputPorts = getReusableTemplateInputPorts();
    Set<ConnectionDTO> reusableTemplateConnections = getReusableTemplateConnections();
    if (componentOption != null && !componentOption.getConnectionInfo().isEmpty()) {
        // connect the output port to the input port
        // we are connecting a reusable template back to another reusable template
        // follow the input port destination connection to its internal process group port
        Set<ConnectionDTO> newConnections = new HashSet<>();
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ReusableTemplateConnectionInfo connectionInfo : componentOption.getConnectionInfo()) {
                String reusableTemplateInputPortName = connectionInfo.getReusableTemplateInputPortName();
                // find the portdto matching this name in the reusable template group
                /**
                 * The connection coming from the 'reusableTemplateInputPortName' to the next input port
                 */
                Optional<ConnectionDTO> connectionToUse = Optional.empty();
                /**
                 * The port that matches the 'reusableTemplateInputPortName connection destination
                 */
                Optional<PortDTO> sourcePort = Optional.empty();
                connectionToUse = inputPorts.stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(reusableTemplateInputPortName)).findFirst().flatMap(portToInspect -> reusableTemplateConnections.stream().filter(connectionDTO -> connectionDTO.getDestination().getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name()) && connectionDTO.getSource().getId().equalsIgnoreCase(portToInspect.getId())).findFirst());
                if (connectionToUse.isPresent()) {
                    sourcePort = newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionInfo.getFeedOutputPortName())).findFirst();
                }
                if (sourcePort.isPresent()) {
                    // make the connection now from the output port to the 'connectionToUse' destination
                    ConnectableDTO source = NifiConnectionUtil.asConnectable(sourcePort.get());
                    ConnectableDTO dest = NifiConnectionUtil.asNewConnectable(connectionToUse.get().getDestination());
                    ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                    newConnections.add(newConnection);
                    connections.add(newConnection);
                    log.info("Connected the output port {} ({}) to another reusable template input port: {} {{}).  The public reusable template port name is: {} ", source.getName(), source.getId(), dest.getName(), dest.getId(), reusableTemplateInputPortName);
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connected this template '" + templateName + "' with '" + reusableTemplateInputPortName + "' connected to '" + sourcePort.get().getName() + "'", true, true);
                } else {
                // log.error("Unable to find a connection to connect the reusable template together.  Please verify the Input Port named '{}' under the 'reusable_templates' group has a connection going to another input port",reusableTemplateInputPortName);
                // importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, "Unable to connect the reusable template to the designated input port: '" + reusableTemplateInputPortName + "'. Please verify the Input Port named '" + reusableTemplateInputPortName + "' under the 'reusable_templates' group has a connection going to another input port.  You may need to re-import the template with this input port. ", "");
                // importStatusMessage.update("Unable to connect the reusable template to the designated input port: "+reusableTemplateInputPortName, false);
                // uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to connect this template '"+templateName+"' with '"+reusableTemplateInputPortName,true,false);
                // break;
                }
            }
        }
        if (!newConnections.isEmpty()) {
            // requery for the ports to check validity again
            processGroupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(newTemplateInstance.getProcessGroupEntity().getId(), false, true).orElse(processGroupDTO);
            // reset it back to the newTemplateInstance
            newTemplateInstance.updateProcessGroupContent(processGroupDTO);
        }
    }
    return processGroupDTO;
}

Example 84 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class NifiRestTest method findFeedProcessGroup.

// @Test
public void findFeedProcessGroup() throws Exception {
    String feedName = "cat_62_feed_282";
    String categoryGroupId = "66e65266-015e-1000-703b-cb619274da55";
    long start = System.currentTimeMillis();
    ProcessGroupDTO feedGroup = restClient.getProcessGroupByName(categoryGroupId, feedName);
    long stop = System.currentTimeMillis();
    long time = stop - start;
    int i = 0;
}

Example 85 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project nifi by apache.

the class StandardNiFiServiceFacade method updateProcessGroup.

@Override
public ProcessGroupEntity updateProcessGroup(final Revision revision, final ProcessGroupDTO processGroupDTO) {
    final ProcessGroup processGroupNode = processGroupDAO.getProcessGroup(processGroupDTO.getId());
    final RevisionUpdate<ProcessGroupDTO> snapshot = updateComponent(revision, processGroupNode, () -> processGroupDAO.updateProcessGroup(processGroupDTO), processGroup -> dtoFactory.createProcessGroupDto(processGroup));
    final PermissionsDTO permissions = dtoFactory.createPermissionsDto(processGroupNode);
    final RevisionDTO updatedRevision = dtoFactory.createRevisionDTO(snapshot.getLastModification());
    final ProcessGroupStatusDTO status = dtoFactory.createConciseProcessGroupStatusDto(controllerFacade.getProcessGroupStatus(processGroupNode.getIdentifier()));
    final List<BulletinDTO> bulletins = dtoFactory.createBulletinDtos(bulletinRepository.findBulletinsForSource(processGroupNode.getIdentifier()));
    final List<BulletinEntity> bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList());
    return entityFactory.createProcessGroupEntity(snapshot.getComponent(), updatedRevision, permissions, status, bulletinEntities);
}