Examples with RemoteProcessGroupInputPort com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort used on opensource projects

Example 1 with RemoteProcessGroupInputPort

use of com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort in project kylo by Teradata.

the class ImportReusableTemplate method validateRemoteInputPorts.

/**
 * Validates the user has supplied some input ports to be created as remote ports
 * @param remoteProcessGroupOption the user supplied option and details for remote process group input port processing
 * @return true if valid, false if not
 */
public boolean validateRemoteInputPorts(ImportComponentOption remoteProcessGroupOption) {
    // find list of input ports that have been created already (connected to this same reusable template)
    // 1) find input ports on parent nifi canvas that connect to the reusable template with this same name
    // 2) add these as 'selected' to the list
    // 3) if some of those dont appear in the new list add as warning (these will be removed)
    boolean valid = true;
    if (!isClustered()) {
        return true;
    }
    // This templates input ports as a map by name
    this.templateInputPorts = importTemplate.getTemplateResults().getProcessGroupEntity().getContents().getInputPorts().stream().collect(Collectors.toMap(p -> p.getName(), v -> v));
    // set the map of input ports in this template as potential Remote Input port candidates.
    this.remoteProcessGroupInputPortMap = this.templateInputPorts.values().stream().map(p -> new RemoteProcessGroupInputPort(importTemplate.getTemplateName(), p.getName())).collect(Collectors.toMap(p -> p.getInputPortName(), p -> p));
    // If the incoming list is empty send it back to the user to validate what input ports they would like (if any) to be created as remote input ports
    if (!remoteProcessGroupOption.isUserAcknowledged()) {
        // present back to the user the list of input ports to select
        importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
        valid = false;
        // WARN if the remoteProcessGroupInputPortMap has names that are not in the 'thisTemplatePorts'
        List<String> invalidPorts = remoteProcessGroupInputPortMap.keySet().stream().filter(name -> !this.templateInputPorts.keySet().contains(name)).collect(Collectors.toList());
        if (!invalidPorts.isEmpty()) {
            // the following ports (invalidPorts) will be deleted from the as they no longer exist for this template.
            // Any remote Process group ports created for them will also be deleted
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, " Missing 'remote process group ' input ports  ", "");
            remoteProcessGroupOption.getErrorMessages().add(" The following 'remote process group ' input ports are no longer part of this template. " + invalidPorts.stream().collect(Collectors.joining(",")) + ". Are you sure you want to continue?  They will be deleted. ");
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, true);
    } else {
        // user has already supplied some ports... validate the ports exist for this template
        // warn if the user supplied input port selections that dont exist for this template
        Set<String> nonExistentPortNames = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> !this.templateInputPorts.keySet().contains(r.getInputPortName())).map(r -> r.getInputPortName()).collect(Collectors.toSet());
        if (!nonExistentPortNames.isEmpty()) {
            importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " Invalid input port names supplied", "");
            remoteProcessGroupOption.getErrorMessages().add("The following input ports you supplied as remote ports dont existing in this template: " + nonExistentPortNames.stream().collect(Collectors.joining(",")) + ".");
            importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
            valid = false;
        }
        valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, false);
    }
    importTemplate.setRemoteProcessGroupInputPortNames(new ArrayList<>(remoteProcessGroupInputPortMap.values()));
    importTemplate.setSuccess(valid);
    importTemplate.setValid(valid);
    return valid;
}

Example 2 with RemoteProcessGroupInputPort

use of com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort in project kylo by Teradata.

the class ImportUtil method openZip.

/**
 * Open the zip file and populate the {@link ImportTemplate} object with the components in the file/archive
 *
 * @param fileName    the file name
 * @param inputStream the file
 * @return the template data to import
 */
public static ImportTemplate openZip(String fileName, InputStream inputStream) throws IOException {
    byte[] buffer = new byte[1024];
    ZipInputStream zis = new ZipInputStream(inputStream);
    ZipEntry zipEntry;
    ImportTemplate importTemplate = new ImportTemplate(fileName);
    while ((zipEntry = zis.getNextEntry()) != null) {
        String zipEntryContents = ZipFileUtil.zipEntryToString(buffer, zis, zipEntry);
        if (zipEntry.getName().startsWith(ImportTemplate.NIFI_TEMPLATE_XML_FILE)) {
            importTemplate.setNifiTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.TEMPLATE_JSON_FILE)) {
            importTemplate.setTemplateJson(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE)) {
            importTemplate.addNifiConnectingReusableTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_REMOTE_INPUT_PORT_JSON_FILE)) {
            String json = zipEntryContents;
            List<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = ObjectMapperSerializer.deserialize(json, new TypeReference<List<RemoteProcessGroupInputPort>>() {
            });
            importTemplate.addRemoteProcessGroupInputPorts(remoteProcessGroupInputPorts);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_OUTPUT_CONNECTION_FILE)) {
            String json = zipEntryContents;
            List<ReusableTemplateConnectionInfo> connectionInfos = ObjectMapperSerializer.deserialize(json, new TypeReference<List<ReusableTemplateConnectionInfo>>() {
            });
            importTemplate.addReusableTemplateConnectionInformation(connectionInfos);
        }
    }
    zis.closeEntry();
    zis.close();
    if (!importTemplate.hasValidComponents()) {
        throw new UnsupportedOperationException(" The file you uploaded is not a valid archive.  Please ensure the Zip file has been exported from the system and has 2 valid files named: " + ImportTemplate.NIFI_TEMPLATE_XML_FILE + ", and " + ImportTemplate.TEMPLATE_JSON_FILE);
    }
    importTemplate.setZipFile(true);
    return importTemplate;
}

Example 3 with RemoteProcessGroupInputPort

use of com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort in project kylo by Teradata.

the class ImportReusableTemplate method validateRemoteInputPorts.

private boolean validateRemoteInputPorts(Set<PortDTO> inputPorts) {
    ImportComponentOption remoteProcessGroupOption = importTemplateOptions.findImportComponentOption(ImportComponent.REMOTE_INPUT_PORT);
    boolean valid = true;
    if (remoteProcessGroupOption.isShouldImport()) {
        // This templates input ports as a map by name
        this.templateInputPorts = inputPorts.stream().collect(Collectors.toMap(p -> p.getName(), v -> v));
        // set the map of input ports in this template as potential Remote Input port candidates.
        this.remoteProcessGroupInputPortMap = this.templateInputPorts.values().stream().map(p -> new RemoteProcessGroupInputPort(importTemplate.getTemplateName(), p.getName())).collect(Collectors.toMap(p -> p.getInputPortName(), p -> p));
        if (!remoteProcessGroupOption.isUserAcknowledged()) {
            // present back to the user the list of input ports to select
            importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
            // WARN if the remoteProcessGroupInputPortMap has names that are not in the 'thisTemplatePorts'
            List<String> invalidPorts = remoteProcessGroupInputPortMap.keySet().stream().filter(name -> !this.templateInputPorts.keySet().contains(name)).collect(Collectors.toList());
            if (!invalidPorts.isEmpty()) {
                // the following ports (invalidPorts) will be deleted from the as they no longer exist for this template.
                // Any remote Process group ports created for them will also be deleted
                importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, " Missing 'remote process group ' input ports  ", "");
                remoteProcessGroupOption.getErrorMessages().add(" The following 'remote process group ' input ports are no longer part of this template. " + invalidPorts.stream().collect(Collectors.joining(",")) + ". Are you sure you want to continue?  They will be deleted. ");
            }
            importTemplate.setRemoteProcessGroupInputPortNames(new ArrayList<>(remoteProcessGroupInputPortMap.values()));
            importTemplate.setSuccess(false);
            importTemplate.setValid(false);
            markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, true);
            valid = false;
        } else {
            // user has already supplied some ports... validate the ports exist for this template
            // warn if the user supplied input port selections that dont exist for this template
            Set<String> nonExistentPortNames = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().filter(r -> !this.templateInputPorts.keySet().contains(r.getInputPortName())).map(r -> r.getInputPortName()).collect(Collectors.toSet());
            if (!nonExistentPortNames.isEmpty()) {
                importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " Invalid input port names supplied", "");
                remoteProcessGroupOption.getErrorMessages().add("The following input ports you supplied as remote ports dont existing in this template: " + nonExistentPortNames.stream().collect(Collectors.joining(",")) + ".");
                importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
                valid = false;
            }
            valid &= markExistingRemoteInputPorts(remoteProcessGroupOption, remoteProcessGroupInputPortMap, this.templateInputPorts, false);
            importTemplate.setSuccess(valid);
            importTemplate.setValid(valid);
        }
    }
    return valid;
}

Example 4 with RemoteProcessGroupInputPort

use of com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort in project kylo by Teradata.

the class ImportReusableTemplate method markExistingRemoteInputPorts.

private boolean markExistingRemoteInputPorts(ImportComponentOption remoteProcessGroupOption, Map<String, RemoteProcessGroupInputPort> remoteProcessGroupInputPortMap, Map<String, PortDTO> thisTemplatePorts, boolean isNew) {
    String rootProcessGroupId = templateConnectionUtil.getRootProcessGroup().getId();
    // This will get or create the reusable_template process group
    String reusableTemplateProcessGroupId = templateConnectionUtil.getReusableTemplateProcessGroupId();
    // Select the userSupplied ports from the whole list
    Map<String, RemoteProcessGroupInputPort> userSuppliedRemoteInputPorts = remoteProcessGroupOption.getRemoteProcessGroupInputPortsForTemplate(importTemplate.getTemplateName()).stream().collect(Collectors.toMap(inputPort -> inputPort.getInputPortName(), inputPort -> inputPort));
    Optional<TemplateRemoteInputPortConnections> existingRemoteProcessInputPortInformation = getExistingRemoteProcessInputPortInformation();
    if (existingRemoteProcessInputPortInformation.isPresent()) {
        // mark the items in the this.remoteProcessGroupInputPortMap as 'existing' if they are already in NiFi, and 'selected' if the user has selected them
        existingRemoteProcessInputPortInformation.get().getExistingRemoteConnectionsToTemplate().stream().filter(conn -> conn.getDestination().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && conn.getDestination().getGroupId().equalsIgnoreCase(reusableTemplateProcessGroupId) && conn.getSource().getGroupId().equalsIgnoreCase(rootProcessGroupId) && conn.getSource().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && thisTemplatePorts.containsKey(conn.getDestination().getName())).map(conn -> thisTemplatePorts.get(conn.getDestination().getName())).filter(p -> remoteProcessGroupInputPortMap.containsKey(p.getName())).map(port -> remoteProcessGroupInputPortMap.get(port.getName())).forEach(remoteProcessGroupInputPort -> {
            remoteProcessGroupInputPort.setSelected(isNew || (userSuppliedRemoteInputPorts.containsKey(remoteProcessGroupInputPort.getInputPortName()) && userSuppliedRemoteInputPorts.get(remoteProcessGroupInputPort.getInputPortName()).isSelected()));
            remoteProcessGroupInputPort.setExisting(true);
            RemoteProcessGroupInputPort userSuppliedPort = userSuppliedRemoteInputPorts.get(remoteProcessGroupInputPort.getInputPortName());
            if (userSuppliedPort != null) {
                userSuppliedPort.setExisting(true);
            }
        });
    }
    remoteProcessGroupInputPortMap.values().stream().filter(inputPort -> userSuppliedRemoteInputPorts.containsKey(inputPort.getInputPortName())).forEach(inputPort -> inputPort.setSelected(true));
    // warn if not existing, but has the same name as an already existing input port in the root process group
    Set<String> rootInputPorts = nifiRestClient.getNiFiRestClient().processGroups().getInputPorts(rootProcessGroupId).stream().map(inputPort -> inputPort.getName()).collect(Collectors.toSet());
    Set<String> portAlreadyExists = remoteProcessGroupInputPortMap.values().stream().filter(remoteProcessGroupInputPort -> !remoteProcessGroupInputPort.isExisting() && rootInputPorts.contains(remoteProcessGroupInputPort.getInputPortName())).map(remoteProcessGroupInputPort -> remoteProcessGroupInputPort.getInputPortName()).collect(Collectors.toSet());
    if (!portAlreadyExists.isEmpty()) {
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, " The input port names " + portAlreadyExists.stream().collect(Collectors.joining(",")) + " already exists as a remote port for another template.", "");
        remoteProcessGroupOption.getErrorMessages().add("The input port names " + portAlreadyExists.stream().collect(Collectors.joining(",")) + " already exists as a remote port for another template.");
        importTemplate.setSuccess(false);
        importTemplate.setValid(false);
        importTemplate.setRemoteProcessGroupInputPortsNeeded(true);
        return false;
    }
    return true;
}

Example 5 with RemoteProcessGroupInputPort

use of com.thinkbiganalytics.feedmgr.rest.model.RemoteProcessGroupInputPort in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = null;
        if (template.usesReusableTemplate()) {
            reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
        }
        List<ReusableTemplateConnectionInfo> remoteProcessGroupConnectionInfo = getRemoteProcessGroupConnectionInfo(template.getNifiTemplate());
        if (reusableTemplateConnectionInfos != null) {
            reusableTemplateConnectionInfos.addAll(remoteProcessGroupConnectionInfo);
        } else {
            reusableTemplateConnectionInfos = remoteProcessGroupConnectionInfo;
        }
        if (reusableTemplateConnectionInfos != null && !reusableTemplateConnectionInfos.isEmpty()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if enabled
            if (isRemoteProcessGroupsEnabled()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (NifiConnectionException e) {
            throw e;
        } catch (Exception e) {
            throw new TemplateExportException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", template.getTemplateName(), template.getDescription(), template.isStream(), zipFile);
    } else {
        throw new TemplateExportException("Unable to find Template for " + templateId);
    }
}