Examples with PortDTOWithGroupInfo com.thinkbiganalytics.feedmgr.rest.model.PortDTOWithGroupInfo used on opensource projects

Example 1 with PortDTOWithGroupInfo

use of com.thinkbiganalytics.feedmgr.rest.model.PortDTOWithGroupInfo in project kylo by Teradata.

the class TemplateConnectionUtil method getReusableFeedInputPorts.

/**
 * @return all input ports under the {@link TemplateCreationHelper#REUSABLE_TEMPLATES_PROCESS_GROUP_NAME} process group
 */
public Set<PortDTOWithGroupInfo> getReusableFeedInputPorts() {
    Set<PortDTOWithGroupInfo> ports = new HashSet<>();
    String reusableProcessGroupId = this.getReusableTemplateProcessGroupId();
    if (reusableProcessGroupId != null) {
        try {
            ProcessGroupFlowDTO processGroup = restClient.getNiFiRestClient().processGroups().flow(reusableProcessGroupId);
            if (processGroup != null) {
                ports.addAll(getReusableFeedInputPorts(processGroup));
            }
        } catch (final NifiComponentNotFoundException e) {
            log.debug("Reusable template process group not found: {}", reusableProcessGroupId, e);
        }
    }
    return ports;
}

Example 2 with PortDTOWithGroupInfo

use of com.thinkbiganalytics.feedmgr.rest.model.PortDTOWithGroupInfo in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getReusableTemplateProcessorsForInputPorts.

/**
 * Return all the processors that are connected to a given NiFi input port
 *
 * @param inputPortIds the ports to inspect
 * @return all the processors that are connected to a given NiFi input port
 */
public List<RegisteredTemplate.Processor> getReusableTemplateProcessorsForInputPorts(List<String> inputPortIds) {
    Set<ProcessorDTO> processorDTOs = new HashSet<>();
    if (inputPortIds != null && !inputPortIds.isEmpty()) {
        ProcessGroupDTO processGroup = nifiRestClient.getProcessGroupByName("root", TemplateCreationHelper.REUSABLE_TEMPLATES_PROCESS_GROUP_NAME);
        if (processGroup != null) {
            // fetch the Content
            ProcessGroupDTO content = nifiRestClient.getProcessGroup(processGroup.getId(), true, true);
            processGroup.setContents(content.getContents());
            Set<PortDTOWithGroupInfo> ports = getReusableFeedInputPorts();
            ports.stream().filter(portDTO -> inputPortIds.contains(portDTO.getId())).forEach(port -> {
                List<ConnectionDTO> connectionDTOs = NifiConnectionUtil.findConnectionsMatchingSourceId(processGroup.getContents().getConnections(), port.getId());
                if (connectionDTOs != null) {
                    connectionDTOs.stream().forEach(connectionDTO -> {
                        String processGroupId = connectionDTO.getDestination().getGroupId();
                        Set<ProcessorDTO> processors = nifiRestClient.getProcessorsForFlow(processGroupId);
                        if (processors != null) {
                            processorDTOs.addAll(processors);
                        }
                    });
                }
            });
        }
    }
    List<RegisteredTemplate.Processor> processorProperties = processorDTOs.stream().map(processorDTO -> registeredTemplateUtil.toRegisteredTemplateProcessor(processorDTO, true)).collect(Collectors.toList());
    return processorProperties;
}

Example 3 with PortDTOWithGroupInfo

use of com.thinkbiganalytics.feedmgr.rest.model.PortDTOWithGroupInfo in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getRegisteredTemplateProcessors.

@Override
public List<RegisteredTemplate.Processor> getRegisteredTemplateProcessors(String templateId, boolean includeReusableProcessors) {
    List<RegisteredTemplate.Processor> processorProperties = new ArrayList<>();
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeAllProperties(true).build());
    if (template != null) {
        template.initializeProcessors();
        processorProperties.addAll(template.getInputProcessors());
        processorProperties.addAll(template.getNonInputProcessors());
        if (includeReusableProcessors && template.getReusableTemplateConnections() != null && !template.getReusableTemplateConnections().isEmpty()) {
            // 1 fetch ports in reusable templates
            Map<String, PortDTO> reusableTemplateInputPorts = new HashMap<>();
            Set<PortDTOWithGroupInfo> ports = getReusableFeedInputPorts();
            if (ports != null) {
                ports.stream().forEach(portDTO -> reusableTemplateInputPorts.put(portDTO.getName(), portDTO));
            }
            // match to the name
            List<String> matchingPortIds = template.getReusableTemplateConnections().stream().filter(conn -> reusableTemplateInputPorts.containsKey(conn.getReusableTemplateInputPortName())).map(reusableTemplateConnectionInfo -> reusableTemplateInputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName()).getId()).collect(Collectors.toList());
            List<RegisteredTemplate.Processor> reusableProcessors = getReusableTemplateProcessorsForInputPorts(matchingPortIds);
            processorProperties.addAll(reusableProcessors);
        }
    }
    return processorProperties;
}

Example 4 with PortDTOWithGroupInfo

use of com.thinkbiganalytics.feedmgr.rest.model.PortDTOWithGroupInfo in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = null;
        if (template.usesReusableTemplate()) {
            reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
        }
        List<ReusableTemplateConnectionInfo> remoteProcessGroupConnectionInfo = getRemoteProcessGroupConnectionInfo(template.getNifiTemplate());
        if (reusableTemplateConnectionInfos != null) {
            reusableTemplateConnectionInfos.addAll(remoteProcessGroupConnectionInfo);
        } else {
            reusableTemplateConnectionInfos = remoteProcessGroupConnectionInfo;
        }
        if (reusableTemplateConnectionInfos != null && !reusableTemplateConnectionInfos.isEmpty()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if enabled
            if (isRemoteProcessGroupsEnabled()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (NifiConnectionException e) {
            throw e;
        } catch (Exception e) {
            throw new TemplateExportException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", template.getTemplateName(), template.getDescription(), template.isStream(), zipFile);
    } else {
        throw new TemplateExportException("Unable to find Template for " + templateId);
    }
}