Examples with ImportComponentOption com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption used on opensource projects

Example 16 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class TemplateImporter method initializeImportTemplateArchive.

private void initializeImportTemplateArchive() throws ImportException {
    try {
        UploadProgress progress = uploadProgressService.getUploadStatus(importOptions.getUploadKey());
        progress.setSections(ImportSection.sectionsForImportAsString(ImportType.TEMPLATE));
        InputStream inputStream = new ByteArrayInputStream(file);
        this.importTemplate = ImportUtil.openZip(fileName, inputStream);
        this.importTemplate.setValid(true);
        Set<ImportComponentOption> componentOptions = ImportUtil.inspectZipComponents(file, ImportType.TEMPLATE);
        importTemplateOptions.addOptionsIfNotExists(componentOptions);
        importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION).addConnectionInfo(importTemplate.getReusableTemplateConnections());
        importTemplateOptions.findImportComponentOption(ImportComponent.REMOTE_INPUT_PORT).addRemoteProcessGroupInputPorts(importTemplate.getRemoteProcessGroupInputPortNames());
        importTemplate.setImportOptions(this.importTemplateOptions);
    } catch (Exception e) {
        throw new ImportException("Unable to open template archive " + fileName + " for import. ", e);
    }
}

Example 17 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class ImportReusableTemplate method validateRemoteInputPorts.

/**
 * Validates the user has supplied some input ports to be created as remote ports
 * @param remoteProcessGroupOption the user supplied option and details for remote process group input port processing
 * @return true if valid, false if not
 */
public boolean validateRemoteInputPorts(ImportComponentOption remoteProcessGroupOption) {
    // find list of input ports that have been created already (connected to this same reusable template)
    // 1) find input ports on parent nifi canvas that connect to the reusable template with this same name
    // 2) add these as 'selected' to the list
    // 3) if some of those dont appear in the new list add as warning (these will be removed)
    boolean valid = true;
    if (!isClustered()) {
        return true;
    }
    // This templates input ports as a map by name
    this.templateInputPorts = importTemplate.getTemplateResults().getProcessGroupEntity().getContents().getInputPorts().stream().collect(Collectors.toMap(p -> p.getName(), v -> v));
    // set the map of input ports in this template as potential Remote Input port candidates.
    this.remoteProcessGroupInputPortMap = this.templateInputPorts.values().stream().map(p -> new RemoteProcessGroupInputPort(importTemplate.getTemplateName(), p.getName())).collect(Collectors.toMap(p -> p.getInputPortName(), p -> p));
    // If the incoming list is empty send it back to the user to validate what input ports they would like (if any) to be created as remote input ports
    if (!remoteProcessGroupOption.isUserAcknowledged()) {
        // present back to the user the list of input ports to select
        importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
        valid = false;
        // WARN if the remoteProcessGroupInputPortMap has names that are not in the 'thisTemplatePorts'
        List<String> invalidPorts = remoteProcessGroupInputPortMap.keySet().stream().filter(name -> !this.templateInputPorts.keySet().contains(name)).collect(Collectors.toList());
        if (!invalidPorts.isEmpty()) {
            // the following ports (invalidPorts) will be deleted from the as they no longer exist for this template.
            // Any remote Process group ports created for them will also be deleted
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, " Missing 'remote process group ' input ports  ", "");
            remoteProcessGroupOption.getErrorMessages().add(" The following 'remote process group ' input ports are no longer part of this template. " + invalidPorts.stream().collect(Collectors.joining(",")) + ". Are you sure you want to continue?  They will be deleted. ");
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, true);
    } else {
        // user has already supplied some ports... validate the ports exist for this template
        // warn if the user supplied input port selections that dont exist for this template
        Set<String> nonExistentPortNames = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> !this.templateInputPorts.keySet().contains(r.getInputPortName())).map(r -> r.getInputPortName()).collect(Collectors.toSet());
        if (!nonExistentPortNames.isEmpty()) {
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " Invalid input port names supplied", "");
            remoteProcessGroupOption.getErrorMessages().add("The following input ports you supplied as remote ports dont existing in this template: " + nonExistentPortNames.stream().collect(Collectors.joining(",")) + ".");
            importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
            valid = false;
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, false);
    }
    importTemplate.setRemoteProcessGroupInputPortNames(new ArrayList<>(remoteProcessGroupInputPortMap.values()));
    importTemplate.setSuccess(valid);
    importTemplate.setValid(valid);
    return valid;
}

Example 18 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class ImportReusableTemplate method connectReusableFlow.

private ProcessGroupDTO connectReusableFlow(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    String templateName = this.importTemplate.getTemplateName();
    ImportComponentOption componentOption = importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION);
    ProcessGroupDTO processGroupDTO = newTemplateInstance.getProcessGroupEntity();
    Set<PortDTO> inputPorts = getReusableTemplateInputPorts();
    Set<ConnectionDTO> reusableTemplateConnections = getReusableTemplateConnections();
    if (componentOption != null && !componentOption.getConnectionInfo().isEmpty()) {
        // connect the output port to the input port
        // we are connecting a reusable template back to another reusable template
        // follow the input port destination connection to its internal process group port
        Set<ConnectionDTO> newConnections = new HashSet<>();
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ReusableTemplateConnectionInfo connectionInfo : componentOption.getConnectionInfo()) {
                String reusableTemplateInputPortName = connectionInfo.getReusableTemplateInputPortName();
                // find the portdto matching this name in the reusable template group
                /**
                 * The connection coming from the 'reusableTemplateInputPortName' to the next input port
                 */
                Optional<ConnectionDTO> connectionToUse = Optional.empty();
                /**
                 * The port that matches the 'reusableTemplateInputPortName connection destination
                 */
                Optional<PortDTO> sourcePort = Optional.empty();
                connectionToUse = inputPorts.stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(reusableTemplateInputPortName)).findFirst().flatMap(portToInspect -> reusableTemplateConnections.stream().filter(connectionDTO -> connectionDTO.getDestination().getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name()) && connectionDTO.getSource().getId().equalsIgnoreCase(portToInspect.getId())).findFirst());
                if (connectionToUse.isPresent()) {
                    sourcePort = newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionInfo.getFeedOutputPortName())).findFirst();
                }
                if (sourcePort.isPresent()) {
                    // make the connection now from the output port to the 'connectionToUse' destination
                    ConnectableDTO source = NifiConnectionUtil.asConnectable(sourcePort.get());
                    ConnectableDTO dest = NifiConnectionUtil.asNewConnectable(connectionToUse.get().getDestination());
                    ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                    newConnections.add(newConnection);
                    connections.add(newConnection);
                    log.info("Connected the output port {} ({}) to another reusable template input port: {} {{}).  The public reusable template port name is: {} ", source.getName(), source.getId(), dest.getName(), dest.getId(), reusableTemplateInputPortName);
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connected this template '" + templateName + "' with '" + reusableTemplateInputPortName + "' connected to '" + sourcePort.get().getName() + "'", true, true);
                } else {
                // log.error("Unable to find a connection to connect the reusable template together.  Please verify the Input Port named '{}' under the 'reusable_templates' group has a connection going to another input port",reusableTemplateInputPortName);
                // importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, "Unable to connect the reusable template to the designated input port: '" + reusableTemplateInputPortName + "'. Please verify the Input Port named '" + reusableTemplateInputPortName + "' under the 'reusable_templates' group has a connection going to another input port.  You may need to re-import the template with this input port. ", "");
                // importStatusMessage.update("Unable to connect the reusable template to the designated input port: "+reusableTemplateInputPortName, false);
                // uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to connect this template '"+templateName+"' with '"+reusableTemplateInputPortName,true,false);
                // break;
                }
            }
        }
        if (!newConnections.isEmpty()) {
            // requery for the ports to check validity again
            processGroupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(newTemplateInstance.getProcessGroupEntity().getId(), false, true).orElse(processGroupDTO);
            // reset it back to the newTemplateInstance
            newTemplateInstance.updateProcessGroupContent(processGroupDTO);
        }
    }
    return processGroupDTO;
}

Example 19 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class ImportReusableTemplate method markExistingRemoteInputPorts.

private boolean markExistingRemoteInputPorts(ImportComponentOption remoteProcessGroupOption, Map<String, RemoteProcessGroupInputPort> remoteProcessGroupInputPortMap, Map<String, PortDTO> thisTemplatePorts, boolean isNew) {
    String rootProcessGroupId = templateConnectionUtil.getRootProcessGroup().getId();
    // This will get or create the reusable_template process group
    String reusableTemplateProcessGroupId = templateConnectionUtil.getReusableTemplateProcessGroupId();
    // Select the userSupplied ports from the whole list
    Map<String, RemoteProcessGroupInputPort> userSuppliedRemoteInputPorts = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().collect(Collectors.toMap(inputPort -> inputPort.getInputPortName(), inputPort -> inputPort));
    Optional<TemplateRemoteInputPortConnections> existingRemoteProcessInputPortInformation = getExistingRemoteProcessInputPortInformation();
    if (existingRemoteProcessInputPortInformation.isPresent()) {
        // mark the items in the this.remoteProcessGroupInputPortMap as 'existing' if they are already in NiFi, and 'selected' if the user has selected them
        existingRemoteProcessInputPortInformation.get().getExistingRemoteConnectionsToTemplate().stream().filter(conn -> conn.getDestination().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && conn.getDestination().getGroupId().equalsIgnoreCase(reusableTemplateProcessGroupId) && conn.getSource().getGroupId().equalsIgnoreCase(rootProcessGroupId) && conn.getSource().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && thisTemplatePorts.containsKey(conn.getDestination().getName())).map(conn -> thisTemplatePorts.get(conn.getDestination().getName())).filter(p -> remoteProcessGroupInputPortMap.containsKey(p.getName())).map(port -> remoteProcessGroupInputPortMap.get(port.getName())).forEach(remoteProcessGroupInputPort -> {
            remoteProcessGroupInputPort.setSelected(isNew || (userSuppliedRemoteInputPorts.containsKey(remoteProcessGroupInputPort.getInputPortName()) && userSuppliedRemoteInputPorts.get(remoteProcessGroupInputPort.getInputPortName()).isSelected()));
            remoteProcessGroupInputPort.setExisting(true);
            RemoteProcessGroupInputPort userSuppliedPort = userSuppliedRemoteInputPorts.get(remoteProcessGroupInputPort.getInputPortName());
            if (userSuppliedPort != null) {
                userSuppliedPort.setExisting(true);
            }
        });
    }
    remoteProcessGroupInputPortMap.values().stream().filter(inputPort -> userSuppliedRemoteInputPorts.containsKey(inputPort.getInputPortName())).forEach(inputPort -> inputPort.setSelected(true));
    // warn if not existing, but has the same name as an already existing input port in the root process group
    Set<String> rootInputPorts = nifiRestClient.getNiFiRestClient().processGroups().getInputPorts(rootProcessGroupId).stream().map(inputPort -> inputPort.getName()).collect(Collectors.toSet());
    Set<String> portAlreadyExists = remoteProcessGroupInputPortMap.values().stream().filter(remoteProcessGroupInputPort -> !remoteProcessGroupInputPort.isExisting() && rootInputPorts.contains(remoteProcessGroupInputPort.getInputPortName())).map(remoteProcessGroupInputPort -> remoteProcessGroupInputPort.getInputPortName()).collect(Collectors.toSet());
    if (!portAlreadyExists.isEmpty()) {
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " The input port names " + portAlreadyExists.stream().collect(Collectors.joining(",")) + " already exists as a remote port for another template.", "");
        remoteProcessGroupOption.getErrorMessages().add("The input port names " + portAlreadyExists.stream().collect(Collectors.joining(",")) + " already exists as a remote port for another template.");
        importTemplate.setSuccess(false);
        importTemplate.setValid(false);
        importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
        return false;
    }
    return true;
}

Example 20 with ImportComponentOption

use of com.thinkbiganalytics.feedmgr.rest.model.ImportComponentOption in project kylo by Teradata.

the class ImportTemplateArchive method importReusableTemplateInArchive.

/**
 * Called when another Template or Feed uses a Reusable template
 */
private List<ImportReusableTemplate> importReusableTemplateInArchive(ImportTemplate importTemplate, ImportTemplateOptions importOptions) {
    List<ImportReusableTemplate> connectingTemplates = new ArrayList<>();
    // start the import
    ImportComponentOption reusableComponentOption = importOptions.findImportComponentOption(ImportComponent.REUSABLE_TEMPLATE);
    if (importTemplate.hasConnectingReusableTemplate() && reusableComponentOption.isShouldImport() && reusableComponentOption.isValidForImport()) {
        UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importOptions.getUploadKey(), "Import Reusable Template. Starting import");
        // import the reusable templates
        boolean valid = true;
        ImportTemplate lastReusableTemplate = null;
        log.info("Importing Zip file template {}. first importing reusable flow from zip");
        try {
            for (String reusableTemplateXml : importTemplate.getNifiConnectingReusableTemplateXmls()) {
                String name = NifiTemplateParser.getTemplateName(reusableTemplateXml);
                byte[] content = reusableTemplateXml.getBytes("UTF-8");
                ImportReusableTemplate importReusableTemplate = importReusableTemplateFactory.apply(name, content, importOptions);
                boolean validReusableTemplate = importReusableTemplate.importIntoNiFiAndCreateInstance();
                connectingTemplates.add(importReusableTemplate);
                if (!validReusableTemplate) {
                    ImportTemplate connectingTemplate = importReusableTemplate.importTemplate;
                    // add in the error messages
                    connectingTemplate.getTemplateResults().getAllErrors().stream().forEach(nifiError -> {
                        importTemplate.getTemplateResults().addError(nifiError);
                    });
                    // error out
                    importTemplate.setSuccess(false);
                    reusableComponentOption.getErrorMessages().add("Error importing Reusable Template");
                    // exit
                    valid = false;
                    break;
                }
            }
            if (valid) {
                statusMessage.update("Successfully imported Reusable Templates " + (connectingTemplates.stream().map(t -> t.getImportTemplate().getTemplateName()).collect(Collectors.joining(","))), true);
            } else {
                statusMessage.update("Errors importing reusable template: Imported Reusable Template. " + lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            }
        } catch (Exception e) {
            log.error("Error importing reusable template from archive {}.  {} ", importTemplate.getFileName(), lastReusableTemplate != null ? lastReusableTemplate.getTemplateName() : "");
            importTemplate.setSuccess(false);
        }
    }
    uploadProgressService.completeSection(importOptions, ImportSection.Section.IMPORT_REUSABLE_TEMPLATE);
    return connectingTemplates;
}