Examples with ReusableTemplateConnectionInfo com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo used on opensource projects

Example 6 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getRegisteredTemplateProcessors.

@Override
public List<RegisteredTemplate.Processor> getRegisteredTemplateProcessors(String templateId, boolean includeReusableProcessors) {
    List<RegisteredTemplate.Processor> processorProperties = new ArrayList<>();
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeAllProperties(true).build());
    if (template != null) {
        template.initializeProcessors();
        processorProperties.addAll(template.getInputProcessors());
        processorProperties.addAll(template.getNonInputProcessors());
        if (includeReusableProcessors && template.getReusableTemplateConnections() != null && !template.getReusableTemplateConnections().isEmpty()) {
            // 1 fetch ports in reusable templates
            Map<String, PortDTO> reusableTemplateInputPorts = new HashMap<>();
            Set<PortDTOWithGroupInfo> ports = getReusableFeedInputPorts();
            if (ports != null) {
                ports.stream().forEach(portDTO -> reusableTemplateInputPorts.put(portDTO.getName(), portDTO));
            }
            // match to the name
            List<String> matchingPortIds = template.getReusableTemplateConnections().stream().filter(conn -> reusableTemplateInputPorts.containsKey(conn.getReusableTemplateInputPortName())).map(reusableTemplateConnectionInfo -> reusableTemplateInputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName()).getId()).collect(Collectors.toList());
            List<RegisteredTemplate.Processor> reusableProcessors = getReusableTemplateProcessorsForInputPorts(matchingPortIds);
            processorProperties.addAll(reusableProcessors);
        }
    }
    return processorProperties;
}

Example 7 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class RegisteredTemplateService method getRegisteredTemplateForUpdate.

/**
 * Return a registered template object that is populated for use with updating in Kylo
 *
 * @param registeredTemplateRequest the request to get a registered template
 * @return a RegisteredTemplate object mapping either one already defined in Kylo, or a new template that maps to one in NiFi
 */
public RegisteredTemplate getRegisteredTemplateForUpdate(RegisteredTemplateRequest registeredTemplateRequest) {
    if (registeredTemplateRequest.isTemplateEdit()) {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_TEMPLATES);
    }
    RegisteredTemplate registeredTemplate = null;
    // attempt to find the template as a Service
    RegisteredTemplateRequest serviceLevelRequest = new RegisteredTemplateRequest(registeredTemplateRequest);
    // editing a feed will run as a service account
    serviceLevelRequest.setFeedEdit(true);
    RegisteredTemplate template = findRegisteredTemplate(serviceLevelRequest);
    boolean canEdit = true;
    if (template != null && StringUtils.isNotBlank(template.getId()) && registeredTemplateRequest.isTemplateEdit()) {
        canEdit = checkTemplatePermission(template.getId(), TemplateAccessControl.EDIT_TEMPLATE);
    }
    if (canEdit) {
        registeredTemplate = template;
        if (registeredTemplate == null) {
            registeredTemplate = nifiTemplateToRegisteredTemplate(registeredTemplateRequest.getNifiTemplateId());
        }
        if (registeredTemplate == null) {
        // throw exception
        } else {
            if (StringUtils.isBlank(registeredTemplate.getId()) && template != null && StringUtils.isNotBlank(template.getId())) {
                registeredTemplate.setId(template.getId());
            }
            Set<PortDTO> ports = null;
            // fetch ports for this template
            try {
                if (registeredTemplate.getNifiTemplate() != null) {
                    ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplate());
                } else {
                    ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplateId());
                }
            } catch (NifiComponentNotFoundException notFoundException) {
                syncNiFiTemplateId(registeredTemplate);
                ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplateId());
            }
            if (ports == null) {
                ports = new HashSet<>();
            }
            List<PortDTO> outputPorts = ports.stream().filter(portDTO -> portDTO != null && NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name().equalsIgnoreCase(portDTO.getType())).collect(Collectors.toList());
            List<PortDTO> inputPorts = ports.stream().filter(portDTO -> portDTO != null && NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name().equalsIgnoreCase(portDTO.getType())).collect(Collectors.toList());
            registeredTemplate.setReusableTemplate(inputPorts != null && !inputPorts.isEmpty());
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = registeredTemplate.getReusableTemplateConnections();
            List<ReusableTemplateConnectionInfo> updatedConnectionInfo = new ArrayList<>();
            for (final PortDTO port : outputPorts) {
                ReusableTemplateConnectionInfo reusableTemplateConnectionInfo = null;
                if (reusableTemplateConnectionInfos != null && !reusableTemplateConnectionInfos.isEmpty()) {
                    reusableTemplateConnectionInfo = Iterables.tryFind(reusableTemplateConnectionInfos, reusableTemplateConnectionInfo1 -> reusableTemplateConnectionInfo1.getFeedOutputPortName().equalsIgnoreCase(port.getName())).orNull();
                }
                if (reusableTemplateConnectionInfo == null) {
                    reusableTemplateConnectionInfo = new ReusableTemplateConnectionInfo();
                    reusableTemplateConnectionInfo.setFeedOutputPortName(port.getName());
                }
                updatedConnectionInfo.add(reusableTemplateConnectionInfo);
            }
            registeredTemplate.setReusableTemplateConnections(updatedConnectionInfo);
            registeredTemplate.initializeProcessors();
            ensureRegisteredTemplateInputProcessors(registeredTemplate);
        }
    }
    return registeredTemplate;
}

Example 8 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        if (template.usesReusableTemplate()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if we are clustered
            NiFiClusterSummary clusterSummary = nifiRestClient.getNiFiRestClient().clusterSummary();
            if (clusterSummary.getClustered()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (Exception e) {
            throw new UnsupportedOperationException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", zipFile);
    } else {
        throw new UnsupportedOperationException("Unable to find Template for " + templateId);
    }
}

Example 9 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class ImportReusableTemplate method connectReusableFlow.

private ProcessGroupDTO connectReusableFlow(UploadProgressMessage importStatusMessage, NifiProcessGroup newTemplateInstance) {
    String templateName = this.importTemplate.getTemplateName();
    ImportComponentOption componentOption = importTemplateOptions.findImportComponentOption(ImportComponent.TEMPLATE_CONNECTION_INFORMATION);
    ProcessGroupDTO processGroupDTO = newTemplateInstance.getProcessGroupEntity();
    Set<PortDTO> inputPorts = getReusableTemplateInputPorts();
    Set<ConnectionDTO> reusableTemplateConnections = getReusableTemplateConnections();
    if (componentOption != null && !componentOption.getConnectionInfo().isEmpty()) {
        // connect the output port to the input port
        // we are connecting a reusable template back to another reusable template
        // follow the input port destination connection to its internal process group port
        Set<ConnectionDTO> newConnections = new HashSet<>();
        String reusableTemplateProcessGroupId = getReusableTemplatesProcessGroupId();
        if (reusableTemplateProcessGroupId != null) {
            for (ReusableTemplateConnectionInfo connectionInfo : componentOption.getConnectionInfo()) {
                String reusableTemplateInputPortName = connectionInfo.getReusableTemplateInputPortName();
                // find the portdto matching this name in the reusable template group
                /**
                 * The connection coming from the 'reusableTemplateInputPortName' to the next input port
                 */
                Optional<ConnectionDTO> connectionToUse = Optional.empty();
                /**
                 * The port that matches the 'reusableTemplateInputPortName connection destination
                 */
                Optional<PortDTO> sourcePort = Optional.empty();
                connectionToUse = inputPorts.stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(reusableTemplateInputPortName)).findFirst().flatMap(portToInspect -> reusableTemplateConnections.stream().filter(connectionDTO -> connectionDTO.getDestination().getType().equalsIgnoreCase(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name()) && connectionDTO.getSource().getId().equalsIgnoreCase(portToInspect.getId())).findFirst());
                if (connectionToUse.isPresent()) {
                    sourcePort = newTemplateInstance.getProcessGroupEntity().getContents().getOutputPorts().stream().filter(portDTO -> portDTO.getName().equalsIgnoreCase(connectionInfo.getFeedOutputPortName())).findFirst();
                }
                if (sourcePort.isPresent()) {
                    // make the connection now from the output port to the 'connectionToUse' destination
                    ConnectableDTO source = NifiConnectionUtil.asConnectable(sourcePort.get());
                    ConnectableDTO dest = NifiConnectionUtil.asNewConnectable(connectionToUse.get().getDestination());
                    ConnectionDTO newConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(reusableTemplateProcessGroupId, source, dest);
                    newConnections.add(newConnection);
                    connections.add(newConnection);
                    log.info("Connected the output port {} ({}) to another reusable template input port: {} {{}).  The public reusable template port name is: {} ", source.getName(), source.getId(), dest.getName(), dest.getId(), reusableTemplateInputPortName);
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Connected this template '" + templateName + "' with '" + reusableTemplateInputPortName + "' connected to '" + sourcePort.get().getName() + "'", true, true);
                } else {
                // log.error("Unable to find a connection to connect the reusable template together.  Please verify the Input Port named '{}' under the 'reusable_templates' group has a connection going to another input port",reusableTemplateInputPortName);
                // importTemplate.getTemplateResults().addError(NifiError.SEVERITY.FATAL, "Unable to connect the reusable template to the designated input port: '" + reusableTemplateInputPortName + "'. Please verify the Input Port named '" + reusableTemplateInputPortName + "' under the 'reusable_templates' group has a connection going to another input port.  You may need to re-import the template with this input port. ", "");
                // importStatusMessage.update("Unable to connect the reusable template to the designated input port: "+reusableTemplateInputPortName, false);
                // uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Unable to connect this template '"+templateName+"' with '"+reusableTemplateInputPortName,true,false);
                // break;
                }
            }
        }
        if (!newConnections.isEmpty()) {
            // requery for the ports to check validity again
            processGroupDTO = nifiRestClient.getNiFiRestClient().processGroups().findById(newTemplateInstance.getProcessGroupEntity().getId(), false, true).orElse(processGroupDTO);
            // reset it back to the newTemplateInstance
            newTemplateInstance.updateProcessGroupContent(processGroupDTO);
        }
    }
    return processGroupDTO;
}

Example 10 with ReusableTemplateConnectionInfo

use of com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo in project kylo by Teradata.

the class ImportUtil method openZip.

/**
 * Open the zip file and populate the {@link ImportTemplate} object with the components in the file/archive
 *
 * @param fileName    the file name
 * @param inputStream the file
 * @return the template data to import
 */
public static ImportTemplate openZip(String fileName, InputStream inputStream) throws IOException {
    byte[] buffer = new byte[1024];
    ZipInputStream zis = new ZipInputStream(inputStream);
    ZipEntry zipEntry;
    ImportTemplate importTemplate = new ImportTemplate(fileName);
    while ((zipEntry = zis.getNextEntry()) != null) {
        String zipEntryContents = ZipFileUtil.zipEntryToString(buffer, zis, zipEntry);
        if (zipEntry.getName().startsWith(ImportTemplate.NIFI_TEMPLATE_XML_FILE)) {
            importTemplate.setNifiTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.TEMPLATE_JSON_FILE)) {
            importTemplate.setTemplateJson(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.NIFI_CONNECTING_REUSABLE_TEMPLATE_XML_FILE)) {
            importTemplate.addNifiConnectingReusableTemplateXml(zipEntryContents);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_REMOTE_INPUT_PORT_JSON_FILE)) {
            String json = zipEntryContents;
            List<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = ObjectMapperSerializer.deserialize(json, new TypeReference<List<RemoteProcessGroupInputPort>>() {
            });
            importTemplate.addRemoteProcessGroupInputPorts(remoteProcessGroupInputPorts);
        } else if (zipEntry.getName().startsWith(ImportTemplate.REUSABLE_TEMPLATE_OUTPUT_CONNECTION_FILE)) {
            String json = zipEntryContents;
            List<ReusableTemplateConnectionInfo> connectionInfos = ObjectMapperSerializer.deserialize(json, new TypeReference<List<ReusableTemplateConnectionInfo>>() {
            });
            importTemplate.addReusableTemplateConnectionInformation(connectionInfos);
        }
    }
    zis.closeEntry();
    zis.close();
    if (!importTemplate.hasValidComponents()) {
        throw new UnsupportedOperationException(" The file you uploaded is not a valid archive.  Please ensure the Zip file has been exported from the system and has 2 valid files named: " + ImportTemplate.NIFI_TEMPLATE_XML_FILE + ", and " + ImportTemplate.TEMPLATE_JSON_FILE);
    }
    importTemplate.setZipFile(true);
    return importTemplate;
}