Examples with VersionedProcessGroup com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup used on opensource projects

Example 1 with VersionedProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup in project kylo by Teradata.

the class ImportReusableTemplate method recreateOutputPortConnections.

private void recreateOutputPortConnections(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    VersionedProcessGroup versionedProcessGroup = newTemplateInstance.getVersionedProcessGroup();
    String templateName = importTemplate.getTemplateName();
    // Dest == input port in versioned off process group
    if (versionedProcessGroup != null) {
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ConnectionDTO connectionDTO : versionedProcessGroup.getDeletedInputPortConnections()) {
                if (connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)) {
                    // connect
                    PortDTO destPort = newTemplateInstance.getProcessGroupEntity().getContents().getInputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionDTO.getDestination().getName()) && connectionDTO.getDestination().getGroupId().equalsIgnoreCase(newTemplateInstance.getVersionedProcessGroup().getProcessGroupPriorToVersioning().getId())).findFirst().orElse(null);
                    if (destPort != null) {
                        // make the connection now from the output port to the 'connectionToUse' destination
                        ConnectableDTO source = NifiConnectionUtil.asNewConnectable(connectionDTO.getSource());
                        ConnectableDTO dest = NifiConnectionUtil.asConnectable(destPort);
                        ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                        connections.add(newConnection);
                        // possibly store the ports too?
                        log.info("Reconnected output port {} ({}) to this new process group input port:  {} {{}) ", source.getName(), source.getId(), dest.getName(), dest.getId());
                    } else {
                        // ERROR cant recreate previous connections that were going into this reusable template
                        String msg = "Unable to recreate the connection for template: " + templateName + " that was previously connected to this template prior to the update. The following connection is missing:  Connecting ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "'].";
                        log.error(msg);
                        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, msg, "");
                        importStatusMessage.update("Unable to establish prior connection for reusable template: " + templateName + ".  Connection:  ['" + connectionDTO.getSource().getName() + "' to '" + connectionDTO.getDestination().getName() + "']", false);
                        break;
                    }
                }
            }
        }
    }
}

Example 2 with VersionedProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup in project kylo by Teradata.

the class TemplateCreationHelper method versionProcessGroup.

/**
 * Version a ProcessGroup renaming it with the name - {timestamp millis}.
 * If {@code removeIfInactive} is true it will not version but just delete it
 *
 * @param processGroup the group to version
 */
public VersionedProcessGroup versionProcessGroup(ProcessGroupDTO processGroup, String versionIdentifier) {
    log.info("Versioning Process Group {} ", processGroup.getName());
    VersionedProcessGroup versionedProcessGroup = new VersionedProcessGroup();
    versionedProcessGroup.setProcessGroupPriorToVersioning(processGroup);
    versionedProcessGroup.setProcessGroupName(processGroup.getName());
    List<ProcessorDTO> inputProcessorsPriorToDisabling = restClient.disableAllInputProcessors(processGroup.getId());
    versionedProcessGroup.setInputProcessorsPriorToDisabling(inputProcessorsPriorToDisabling);
    log.info("Disabled Inputs for {} ", processGroup.getName());
    // attempt to stop all processors
    try {
        restClient.stopInputs(processGroup.getId());
        log.info("Stopped Input Ports for {}, ", processGroup.getName());
    } catch (Exception e) {
        log.error("Error trying to stop Input Ports for {} while creating a new version ", processGroup.getName());
    }
    // delete input connections
    try {
        List<ConnectionDTO> deletedConnections = deleteInputPortConnections(processGroup);
        versionedProcessGroup.setDeletedInputPortConnections(deletedConnections);
    } catch (NifiClientRuntimeException e) {
        log.error("Error trying to delete input port connections for Process Group {} while creating a new version. ", processGroup.getName(), e);
        getErrors().add(new NifiError(NifiError.SEVERITY.FATAL, "The input port connections to the process group " + processGroup.getName() + " could not be deleted. Please delete them manually " + "in NiFi and try again."));
    }
    String versionedProcessGroupName = getVersionedProcessGroupName(processGroup.getName(), versionIdentifier);
    versionedProcessGroup.setVersionedProcessGroupName(versionedProcessGroupName);
    // rename the feedGroup to be name+timestamp
    processGroup.setName(versionedProcessGroupName);
    restClient.updateProcessGroup(processGroup);
    log.info("Renamed ProcessGroup to  {}, ", processGroup.getName());
    versionedProcessGroup.setVersionedProcessGroup(processGroup);
    return versionedProcessGroup;
}

Example 3 with VersionedProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup in project kylo by Teradata.

the class ImportReusableTemplate method cleanup.

public void cleanup() {
    if (newTemplateInstance != null) {
        String templateName = importTemplate.getTemplateName();
        VersionedProcessGroup versionedProcessGroup = newTemplateInstance.getVersionedProcessGroup();
        if (versionedProcessGroup != null) {
            // ensure we have contents in the versionedProcess group
            if (versionedProcessGroup.getVersionedProcessGroup().getContents() == null) {
                FlowDTO flowDTO = nifiRestClient.getNiFiRestClient().processGroups().flow(versionedProcessGroup.getVersionedProcessGroup().getId()).getFlow();
                if (flowDTO != null) {
                    versionedProcessGroup.getVersionedProcessGroup().setContents(NifiFlowUtil.flowToFlowSnippet(flowDTO));
                }
            }
            // importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup()
            uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "The Reusable template " + templateName + " is valid.  Attempting to clean up and remove the previous instance ", true, true);
            // if the versioned group doesnt have anything in queue delete it
            Optional<ProcessGroupStatusDTO> status = nifiRestClient.getNiFiRestClient().processGroups().getStatus(versionedProcessGroup.getVersionedProcessGroup().getId());
            if (!status.isPresent() || (status.isPresent() && status.get().getAggregateSnapshot().getFlowFilesQueued() > 0)) {
                uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Warning! The previous version of this template '" + templateName + "', now located in NiFi under the process group '" + versionedProcessGroup.getVersionedProcessGroupName() + "' still has items in queue.  Unable to delete it. ", true, true);
                // if the versioned group still had items in queue then wire it back up to the newly created group
                // ?? are they still wired up to the output ports
                log.info("Unable to delete versioned group '{}'  Items are still in queue", versionedProcessGroup.getVersionedProcessGroupName());
                // stop the inputs and mark the rest as running
                versionedProcessGroup.getVersionedProcessGroup().getContents().getInputPorts().stream().forEach(portDTO -> {
                    nifiRestClient.stopInputPort(versionedProcessGroup.getVersionedProcessGroup().getId(), portDTO.getId());
                });
                // start the output ports
                versionedProcessGroup.getVersionedProcessGroup().getContents().getOutputPorts().stream().forEach(portDTO -> {
                    nifiRestClient.startOutputPort(versionedProcessGroup.getVersionedProcessGroup().getId(), portDTO.getId());
                });
            } else {
                // delete the versioned group
                try {
                    nifiRestClient.removeProcessGroup(importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup().getId(), importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup().getParentGroupId());
                } catch (NifiClientRuntimeException e) {
                    log.error("Template has sucessfully imported, but Kylo is unable to remove versioned process group '{}' with id '{}'.  You will need to go into NiFi and manually remove it.", versionedProcessGroup.getVersionedProcessGroupName(), versionedProcessGroup.getVersionedProcessGroup().getId());
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Warning! Unable to remove the previous version of this template, process group: '" + versionedProcessGroup.getVersionedProcessGroupName() + "' with id of " + versionedProcessGroup.getVersionedProcessGroup().getId() + ".  You will need to manually delete it from NiFi. ", true, false);
                }
            }
        }
    }
}

Example 4 with VersionedProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup in project kylo by Teradata.

the class TemplateInstanceCreator method createTemplate.

public NifiProcessGroup createTemplate() throws TemplateCreationException {
    try {
        NifiProcessGroup newProcessGroup = null;
        TemplateDTO template = restClient.getTemplateById(templateId);
        VersionedProcessGroup versionedProcessGroup = null;
        if (template != null) {
            TemplateCreationHelper templateCreationHelper = new TemplateCreationHelper(this.restClient);
            templateCreationHelper.setTemplateProperties(templateProperties);
            String processGroupId = null;
            ProcessGroupDTO group = null;
            if (isCreateReusableFlow()) {
                log.info("Creating a new Reusable flow instance for template: {} ", template.getName());
                // 1 get/create the parent "reusable_templates" processgroup
                ProcessGroupDTO reusableParentGroup = restClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                if (reusableParentGroup == null) {
                    reusableParentGroup = restClient.createProcessGroup("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
                }
                ProcessGroupDTO thisGroup = restClient.getProcessGroupByName(reusableParentGroup.getId(), template.getName());
                if (thisGroup != null) {
                    // version the group
                    log.info("A previous Process group of with this name {} was found.  Versioning it before continuing", thisGroup.getName());
                    versionedProcessGroup = templateCreationHelper.versionProcessGroup(thisGroup, this.getVersionIdentifier());
                }
                group = restClient.createProcessGroup(reusableParentGroup.getId(), template.getName());
            } else {
                String tmpName = template.getName() + "_" + System.currentTimeMillis();
                group = restClient.createProcessGroup(tmpName);
                log.info("Creating a temporary process group with name {} for template {} ", tmpName, template.getName());
            }
            processGroupId = group.getId();
            if (StringUtils.isNotBlank(processGroupId)) {
                // snapshot the existing controller services
                templateCreationHelper.snapshotControllerServiceReferences();
                log.info("Successfully Snapshot of controller services");
                // create the flow from the template
                TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
                FlowSnippetDTO flowSnippetDTO = instance.getFlowSnippetDTO();
                log.info("Successfully created the temp flow");
                if (this.createReusableFlow) {
                    ensureInputPortsForReuseableTemplate(processGroupId);
                    log.info("Reusable flow, input ports created successfully.");
                }
                // mark the new services that were created as a result of creating the new flow from the template
                templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
                ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
                // replace static properties and inject values into the flow
                List<NifiProperty> processorProperties = NifiPropertyUtil.getProperties(entity, restClient.getPropertyDescriptorTransform());
                if (processorProperties != null) {
                    boolean didReplace = false;
                    for (NifiProperty property : processorProperties) {
                        boolean replaced = ConfigurationPropertyReplacer.resolveStaticConfigurationProperty(property, staticConfigPropertyMap);
                        if (replaced) {
                            // update the properties that are replaced
                            if (property.getPropertyDescriptor() != null && StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())) {
                                // verify the property is a valid cs property
                                String value = property.getValue();
                                if (templateCreationHelper.getEnabledServiceNameMap().containsKey(value)) {
                                    property.setValue(templateCreationHelper.getEnabledServiceNameMap().get(value).get(0).getId());
                                }
                            }
                            restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property);
                            didReplace = true;
                        }
                    }
                    // if we replaced any properties, refetch to get the latest data
                    if (didReplace) {
                        entity = restClient.getProcessGroup(processGroupId, true, true);
                    }
                }
                // identify the various processors (first level initial processors)
                List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
                ProcessorDTO input = null;
                List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                // if the input is null attempt to get the first input available on the template
                if (input == null && inputProcessors != null && !inputProcessors.isEmpty()) {
                    input = inputProcessors.get(0);
                }
                log.info("Attempt to update/validate controller services for template.");
                // update any references to the controller services and try to assign the value to an enabled service if it is not already
                boolean updatedControllerServices = false;
                if (input != null) {
                    log.info("attempt to update controllerservices on {} input processor ", input.getName());
                    List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(inputProcessors), staticConfigPropertyStringMap, instance);
                    updatedControllerServices = !updatedProperties.isEmpty();
                }
                log.info("attempt to update controllerservices on {} processors ", (nonInputProcessors != null ? nonInputProcessors.size() : 0));
                List<NifiProperty> updatedProperties = templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, staticConfigPropertyStringMap, instance);
                if (!updatedControllerServices) {
                    updatedControllerServices = !updatedProperties.isEmpty();
                }
                log.info("Controller service validation complete");
                // refetch processors for updated errors
                entity = restClient.getProcessGroup(processGroupId, true, true);
                nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                inputProcessors = NifiProcessUtil.getInputProcessors(entity);
                if (inputProcessors != null && !inputProcessors.isEmpty()) {
                    input = inputProcessors.get(0);
                }
                // /make the input/output ports in the category group as running
                if (isCreateReusableFlow()) {
                    log.info("Reusable flow, attempt to mark the connection ports as running.");
                    templateCreationHelper.startProcessGroupAndParentInputPorts(entity);
                    // templateCreationHelper.markConnectionPortsAsRunning(entity);
                    log.info("Reusable flow.  Successfully marked the ports as running.");
                }
                if (isCreateReusableFlow()) {
                    // call listeners notify of before mark as running  processing
                    if (creationCallback != null) {
                        try {
                            creationCallback.beforeMarkAsRunning(template.getName(), entity);
                            // refetch the data for errors
                            entity = restClient.getProcessGroup(processGroupId, true, true);
                            nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                            inputProcessors = NifiProcessUtil.getInputProcessors(entity);
                            if (inputProcessors != null && !inputProcessors.isEmpty()) {
                                input = inputProcessors.get(0);
                            }
                        } catch (Exception e) {
                            log.error("Error calling callback beforeMarkAsRunning ", e);
                        }
                    }
                }
                // create the new process group object that will extract the validation errors from the nifi entity
                newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
                newProcessGroup.setVersionedProcessGroup(versionedProcessGroup);
                newProcessGroup.setReusableFlowInstance(isCreateReusableFlow());
                if (isCreateReusableFlow()) {
                    log.info("Reusable flow, attempt to mark the Processors as running.");
                    templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
                    log.info("Reusable flow.  Successfully marked the Processors as running.");
                    // align items
                    AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
                    alignProcessGroupComponents.autoLayout();
                }
                templateCreationHelper.cleanupControllerServices();
                log.info("Controller service cleanup complete");
                List<NifiError> errors = templateCreationHelper.getErrors();
                // add any global errors to the object
                if (errors != null && !errors.isEmpty()) {
                    for (NifiError error : errors) {
                        newProcessGroup.addError(error);
                    }
                }
                newProcessGroup.setSuccess(!newProcessGroup.hasFatalErrors());
                if (!newProcessGroup.isSuccess()) {
                    log.info("Errors while importing the template. {} errors found. {}", (errors != null ? errors.size() : 0), (errors != null ? " - " + StringUtils.join(errors, ",") : ""));
                } else {
                    log.info("Success.  Finished importing template ");
                }
                return newProcessGroup;
            }
        }
    } catch (final Exception e) {
        if (log.isDebugEnabled() && e instanceof WebApplicationException) {
            final Response response = ((WebApplicationException) e).getResponse();
            log.debug("NiFi server returned error: {}", response.readEntity(String.class), e);
        }
        throw new TemplateCreationException("Unable to create the template for the Id of [" + templateId + "]. " + e.getMessage(), e);
    }
    return null;
}

Example 5 with VersionedProcessGroup

use of com.thinkbiganalytics.nifi.rest.model.VersionedProcessGroup in project kylo by Teradata.

the class ImportReusableTemplate method rollbackTemplateImportInNifi.

/**
 * Restore the previous Template back to Nifi
 */
private void rollbackTemplateImportInNifi() {
    UploadProgressMessage rollbackMessage = restoreOldTemplateXml();
    // If we are working with a reusable flow we need to recreate the old one
    if (importTemplate.getTemplateResults() != null && importTemplate.getTemplateResults().isReusableFlowInstance()) {
        UploadProgressMessage progressMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Attempting to restore old instance for: " + importTemplate.getTemplateName());
        VersionedProcessGroup versionedProcessGroup = null;
        if (importTemplate.getTemplateResults().getVersionedProcessGroup() != null) {
            versionedProcessGroup = importTemplate.getTemplateResults().getVersionedProcessGroup();
        }
        // rename the one we created to a temp name
        ProcessGroupDTO groupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(importTemplate.getTemplateResults().getProcessGroupEntity().getId(), false, false).orElse(null);
        if (groupDTO != null) {
            String tmpName = groupDTO.getName() + "- " + System.currentTimeMillis();
            groupDTO.setName(tmpName);
            nifiRestClient.getNiFiRestClient().processGroups().update(groupDTO);
            log.info("Rollback Template: {}.  Renamed template instance that was just created to a temporary name of {}.  This will get deleted later. ", importTemplate.getTemplateName(), tmpName);
        }
        if (versionedProcessGroup != null) {
            progressMessage.update("Rollback Status: Attempting to initialize and verify prior template instance for " + importTemplate.getTemplateName());
            // rename the versioned one back
            ProcessGroupDTO oldProcessGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(versionedProcessGroup.getProcessGroupPriorToVersioning().getId(), true, true).orElse(null);
            if (oldProcessGroup != null) {
                oldProcessGroup.setName(versionedProcessGroup.getProcessGroupName());
                nifiRestClient.getNiFiRestClient().processGroups().update(oldProcessGroup);
                progressMessage.update("Rollback Status: Renamed template process group " + versionedProcessGroup.getVersionedProcessGroupName() + " back to " + versionedProcessGroup.getProcessGroupName());
            }
            // add back in the connections
            List<ConnectionDTO> createdConnections = new ArrayList<>();
            List<ConnectionDTO> connections = versionedProcessGroup.getDeletedInputPortConnections();
            if (connections != null) {
                connections.stream().forEach(connectionDTO -> {
                    createdConnections.add(nifiRestClient.getNiFiRestClient().processGroups().createConnection(connectionDTO.getParentGroupId(), connectionDTO.getSource(), connectionDTO.getDestination()));
                });
                uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Rollback Status: Recreated " + createdConnections.size() + " connections ", true, true);
            }
            List<ProcessorDTO> inputs = versionedProcessGroup.getInputProcessorsPriorToDisabling();
            if (inputs != null) {
                // update the state
                progressMessage.update("Rollback Status: Marking the process group " + versionedProcessGroup.getProcessGroupName() + " as running");
            }
        }
        if (groupDTO != null) {
            progressMessage.update("Rollback Status: Removing invalid template instance process group:  " + groupDTO.getName());
            // delete the new one
            try {
                nifiRestClient.removeProcessGroup(groupDTO.getId(), groupDTO.getParentGroupId());
            } catch (Exception e) {
                log.error("Error trying to remove invalid template instance {}", groupDTO.getName(), e);
            }
            Optional<ProcessGroupDTO> deletedGroup = nifiRestClient.getNiFiRestClient().processGroups().findById(groupDTO.getId(), false, false);
            if (deletedGroup.isPresent()) {
                progressMessage.update("Rollback Status: Failure", false);
                rollbackMessage.update("Rollback Unsuccessful!!  The invalid group " + deletedGroup.get().getName() + " still exists.  You will need to login to NiFi and verify your reusable templates are correct!", false);
            } else {
                String message = "Rollback Status: Success.";
                if (versionedProcessGroup != null) {
                    message += " Restored '" + versionedProcessGroup.getVersionedProcessGroupName() + "' back to '" + importTemplate.getTemplateName() + "'";
                }
                progressMessage.update(message, true);
                rollbackMessage.update("Rollback Successful!", true);
            }
        }
    } else {
        rollbackMessage.update("Rollback Successful!", true);
    }
}