Examples with UploadProgressMessage com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage used on opensource projects

Example 26 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportReusableTemplate method connectReusableFlow.

private ProcessGroupDTO connectReusableFlow(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    String templateName = this.importTemplate.getTemplateName();
    ImportComponentOption componentOption = importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION);
    ProcessGroupDTO processGroupDTO = newTemplateInstance.getProcessGroupEntity();
    Set<PortDTO> inputPorts = getReusableTemplateInputPorts();
    Set<ConnectionDTO> reusableTemplateConnections = getReusableTemplateConnections();
    if (componentOption != null && !componentOption.getConnectionInfo().isEmpty()) {
        // connect the output port to the input port
        // we are connecting a reusable template back to another reusable template
        // follow the input port destination connection to its internal process group port
        Set<ConnectionDTO> newConnections = new HashSet<>();
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ReusableTemplateConnectionInfo connectionInfo : componentOption.getConnectionInfo()) {
                String reusableTemplateInputPortName = connectionInfo.getReusableTemplateInputPortName();
                // find the portdto matching this name in the reusable template group
                /**
                 * The connection coming from the 'reusableTemplateInputPortName' to the next input port
                 */
                Optional<ConnectionDTO> connectionToUse = Optional.empty();
                /**
                 * The port that matches the 'reusableTemplateInputPortName connection destination
                 */
                Optional<PortDTO> sourcePort = Optional.empty();
                connectionToUse = inputPorts.stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(reusableTemplateInputPortName)).findFirst().flatMap(portToInspect -> reusableTemplateConnections.stream().filter(connectionDTO -> connectionDTO.getDestination().getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name()) && connectionDTO.getSource().getId().equalsIgnoreCase(portToInspect.getId())).findFirst());
                if (connectionToUse.isPresent()) {
                    sourcePort = newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionInfo.getFeedOutputPortName())).findFirst();
                }
                if (sourcePort.isPresent()) {
                    // make the connection now from the output port to the 'connectionToUse' destination
                    ConnectableDTO source = NifiConnectionUtil.asConnectable(sourcePort.get());
                    ConnectableDTO dest = NifiConnectionUtil.asNewConnectable(connectionToUse.get().getDestination());
                    ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                    newConnections.add(newConnection);
                    connections.add(newConnection);
                    log.info("Connected the output port {} ({}) to another reusable template input port: {} {{}).  The public reusable template port name is: {} ", source.getName(), source.getId(), dest.getName(), dest.getId(), reusableTemplateInputPortName);
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connected this template '" + templateName + "' with '" + reusableTemplateInputPortName + "' connected to '" + sourcePort.get().getName() + "'", true, true);
                } else {
                // log.error("Unable to find a connection to connect the reusable template together.  Please verify the Input Port named '{}' under the 'reusable_templates' group has a connection going to another input port",reusableTemplateInputPortName);
                // importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, "Unable to connect the reusable template to the designated input port: '" + reusableTemplateInputPortName + "'. Please verify the Input Port named '" + reusableTemplateInputPortName + "' under the 'reusable_templates' group has a connection going to another input port.  You may need to re-import the template with this input port. ", "");
                // importStatusMessage.update("Unable to connect the reusable template to the designated input port: "+reusableTemplateInputPortName, false);
                // uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to connect this template '"+templateName+"' with '"+reusableTemplateInputPortName,true,false);
                // break;
                }
            }
        }
        if (!newConnections.isEmpty()) {
            // requery for the ports to check validity again
            processGroupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(newTemplateInstance.getProcessGroupEntity().getId(), false, true).orElse(processGroupDTO);
            // reset it back to the newTemplateInstance
            newTemplateInstance.updateProcessGroupContent(processGroupDTO);
        }
    }
    return processGroupDTO;
}

Example 27 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportTemplateArchive method importTemplate.

public boolean importTemplate() {
    UploadProgressMessage statusMessage = null;
    // Get information about the import
    RegisteredTemplate template = importTemplate.getTemplateToImport();
    // validateTemplateProperties(template, importTemplate, importOptions);
    // 1 ensure this template doesnt already exist
    importTemplate.setTemplateName(template.getTemplateName());
    RegisteredTemplate existingTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateName(template.getTemplateName()));
    if (existingTemplate != null) {
        template.setId(existingTemplate.getId());
    } else {
        template.setId(null);
    }
    // first we just import the reusable templates as flows
    this.importedReusableTemplates = importReusableTemplateInArchive(importTemplate, this.importTemplateOptions);
    // after the templates are created we then connect the templates
    if (!this.importTemplateOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE).hasErrorMessages()) {
        if (!importedReusableTemplates.isEmpty()) {
            connectReusableTemplates();
        } else {
            importTemplate.setSuccess(true);
        }
        if (importTemplate.isSuccess()) {
            RegisteredTemplate newTemplate = importFeedTemplate(existingTemplate);
            if (importTemplate.isSuccess()) {
                if (newTemplate != null) {
                    validateInstance();
                }
                if (!importedReusableTemplates.isEmpty()) {
                    importedReusableTemplates.stream().filter(importReusableTemplate -> importReusableTemplate.getImportTemplate().isSuccess()).map(t -> t.getImportTemplate()).forEach(connectingTemplate -> {
                        nifiRestClient.markConnectionPortsAsRunning(connectingTemplate.getTemplateResults().getProcessGroupEntity());
                    });
                }
            }
        }
    } else {
        rollback();
        // return if invalid
        return false;
    }
    // cleanup any temp process groups for this template
    if (!importTemplateOptions.isDeferCleanup()) {
        cleanup();
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.IMPORT_REGISTERED_TEMPLATE);
    return importTemplate.isSuccess();
}

Example 28 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class ImportTemplateArchive method importReusableTemplateInArchive.

/**
 * Called when another Template or Feed uses a Reusable template
 */
private List<ImportReusableTemplate> importReusableTemplateInArchive(ImportTemplate importTemplate, ImportTemplateOptions importOptions) {
    List<ImportReusableTemplate> connectingTemplates = new ArrayList<>();
    // start the import
    ImportComponentOption reusableComponentOption = importOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE);
    if (importTemplate.hasConnectingReusableTemplate() && reusableComponentOption.isShouldImport() && reusableComponentOption.isValidForImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Import Reusable Template. Starting import");
        // import the reusable templates
        boolean valid = true;
        ImportTemplate lastReusableTemplate = null;
        log.info("Importing Zip file template {}. first importing reusable flow from zip");
        try {
            for (String reusableTemplateXml : importTemplate.getNifiConnectingReusableTemplateXmls()) {
                String name = NifiTemplateParser.getTemplateName(reusableTemplateXml);
                byte[] content = reusableTemplateXml.getBytes("UTF-8");
                ImportReusableTemplate importReusableTemplate = importReusableTemplateFactory.apply(name, content, importOptions);
                boolean validReusableTemplate = importReusableTemplate.importIntoNiFiAndCreateInstance();
                connectingTemplates.add(importReusableTemplate);
                if (!validReusableTemplate) {
                    ImportTemplate connectingTemplate = importReusableTemplate.importTemplate;
                    // add in the error messages
                    connectingTemplate.getTemplateResults().getAllErrors().stream().forEach(nifiError -> {
                        importTemplate.getTemplateResults().addError(nifiError);
                    });
                    // error out
                    importTemplate.setSuccess(false);
                    reusableComponentOption.getErrorMessages().add("Error importing Reusable Template");
                    // exit
                    valid = false;
                    break;
                }
            }
            if (valid) {
                statusMessage.update("Successfully imported Reusable Templates " + (connectingTemplates.stream().map(t -> t.getImportTemplate().getTemplateName()).collect(Collectors.joining(","))), true);
            } else {
                statusMessage.update("Errors importing reusable template: Imported Reusable Template. " + lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            }
        } catch (Exception e) {
            log.error("Error importing reusable template from archive {}.  {} ", importTemplate.getFileName(), lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            importTemplate.setSuccess(false);
        }
    }
    uploadProgressService.completeSection(importOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE);
    return connectingTemplates;
}

Example 29 with UploadProgressMessage

use of com.thinkbiganalytics.feedmgr.rest.model.UploadProgressMessage in project kylo by Teradata.

the class AbstractValidateImportTemplate method validateNiFiTemplateImport.

/**
 * Validate the NiFi template is valid.  This method will validate the template can be created/overwritten based upon the user supplied properties
 */
public void validateNiFiTemplateImport() {
    // ImportOptions options = template.getImportOptions();
    ImportComponentOption nifiTemplateOption = this.importTemplateOptions.findImportComponentOption(ImportComponent.NIFI_TEMPLATE);
    // if the options of the TEMPLATE_DATA are marked to import and overwrite this should be as well
    ImportComponentOption templateData = this.importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_DATA);
    if (templateData.isUserAcknowledged()) {
        nifiTemplateOption.setUserAcknowledged(true);
    }
    if (templateData.isShouldImport()) {
        nifiTemplateOption.setShouldImport(true);
    }
    if (templateData.isOverwrite()) {
        nifiTemplateOption.setOverwrite(true);
    }
    if (nifiTemplateOption.isShouldImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Validating the NiFi template");
        String templateName = null;
        TemplateDTO dto = null;
        try {
            templateName = NifiTemplateParser.getTemplateName(this.importTemplate.getNifiTemplateXml());
            this.importTemplate.setTemplateName(templateName);
            dto = nifiRestClient.getNiFiRestClient().templates().findByName(templateName).orElse(null);
            if (dto != null) {
                this.importTemplate.setNifiTemplateId(dto.getId());
                // if the template incoming is an XML template and it already exists, or if its a zip file and it exists and the user has not acknowledge to overwrite then error out
                if ((!this.importTemplateOptions.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) || this.importTemplateOptions.isUserAcknowledged(ImportComponent.NIFI_TEMPLATE) && !this.importTemplate.isZipFile()) && !this.importTemplateOptions.isImportAndOverwrite(ImportComponent.NIFI_TEMPLATE) && !this.importTemplateOptions.isContinueIfExists(ImportComponent.NIFI_TEMPLATE)) {
                    this.importTemplate.setValid(false);
                    String msg = "Unable to import Template " + templateName + ".  It already exists in NiFi.";
                    this.importTemplate.getImportOptions().addErrorMessage(ImportComponent.NIFI_TEMPLATE, msg);
                    statusMessage.update("Validation Error: Unable to import Template " + templateName + ".  It already exists in NiFi.");
                    statusMessage.complete(false);
                } else {
                    statusMessage.update("Validated the NiFi template. ");
                    statusMessage.complete(true);
                }
            } else {
                statusMessage.update("Validated the NiFi template.  The template " + templateName + " will be created in NiFi");
                statusMessage.complete(true);
            }
        } catch (ParserConfigurationException | XPathExpressionException | IOException | SAXException e) {
            getLogger().error("Error validating the file {} for import ", fileName, e);
            this.importTemplate.setValid(false);
            this.importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "The xml file you are trying to import is not a valid NiFi template.  Please try again. " + e.getMessage(), "");
            statusMessage.complete(false);
        }
        nifiTemplateOption.setValidForImport(!nifiTemplateOption.hasErrorMessages());
    }
    uploadProgressService.completeSection(importTemplateOptions, ImportSection.Section.VALIDATE_NIFI_TEMPLATE);
}