Examples with ProcessGroupFlowDTO org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO used on opensource projects

Example 6 with ProcessGroupFlowDTO

use of org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO in project kylo by Teradata.

the class TemplateConnectionUtil method getReusableFeedInputPorts.

/**
 * @return all input ports under the {@link TemplateCreationHelper#REUSABLE_TEMPLATES_PROCESS_GROUP_NAME} process group
 */
public Set<PortDTOWithGroupInfo> getReusableFeedInputPorts() {
    Set<PortDTOWithGroupInfo> ports = new HashSet<>();
    String reusableProcessGroupId = this.getReusableTemplateProcessGroupId();
    if (reusableProcessGroupId != null) {
        ProcessGroupFlowDTO processGroup = restClient.getNiFiRestClient().processGroups().flow(reusableProcessGroupId);
        if (processGroup != null) {
            ports.addAll(getReusableFeedInputPorts(processGroup));
        }
    }
    return ports;
}

Example 7 with ProcessGroupFlowDTO

use of org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO in project kylo by Teradata.

the class NiFiFlowInspector method inspect.

/**
 * Inspects the process group
 *
 * @return the inspection result with the contents of the process group
 */
public NiFiFlowInspection inspect() {
    NiFiFlowInspection inspection = new NiFiFlowInspection(processGroupId, level, parent, Thread.currentThread().getName());
    long start = System.currentTimeMillis();
    if (retryNumber > 0) {
        log.info("Retry inspection attempt number: {}.  Inspecting process group: {} on thread {} ", retryNumber, processGroupId, Thread.currentThread().getName());
    }
    ProcessGroupFlowDTO flow = null;
    try {
        flow = restClient.processGroups().flow(processGroupId);
        if (retryNumber > 0) {
            log.info("Reattempt was successful.  Successfully inspected process group: {} on thread: {} after {} retry attempts. ", processGroupId, Thread.currentThread().getName(), retryNumber);
        }
    } catch (Exception e) {
        // retry
        retryNumber++;
        boolean shouldRetry = retryNumber < RETRIES;
        log.warn("Exception while inspecting process group: {} on thread {}. {} ", processGroupId, Thread.currentThread().getName(), shouldRetry ? "The system will attempt to inspect again." : " out of retry attempts.");
        if (retryNumber < RETRIES) {
            log.warn("Retry inspecting process group: {} on thread {}. ", processGroupId, Thread.currentThread().getName());
            inspect();
        } else {
            log.error("Unable to inspect process group: {} after {} attempts.  Kylo Operations Manager may have issues processing Jobs/Steps from NiFi. ", processGroupId, retryNumber);
        }
    }
    if (flow != null) {
        inspection.setProcessGroupName(flow.getBreadcrumb().getBreadcrumb().getName());
        flow.getFlow().getProcessGroups().stream().forEach(processGroupEntity -> {
            inspection.addGroupToInspect(processGroupEntity.getId());
        });
        inspection.setProcessGroupFlow(flow);
        inspection.setTime(System.currentTimeMillis() - start);
    }
    return inspection;
}

Example 8 with ProcessGroupFlowDTO

use of org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO in project kylo by Teradata.

the class DefaultTemplateExporter method gatherConnectedReusableTemplates.

private void gatherConnectedReusableTemplates(List<String> connectingReusableTemplates, Set<String> connectedTemplateIds, Set<ReusableTemplateConnectionInfo> connectingOutputPortConnectionMetadata, List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos, ProcessGroupFlowDTO reusableTemplateFlow) {
    for (ReusableTemplateConnectionInfo reusableTemplateConnectionInfo : reusableTemplateConnectionInfos) {
        String inputName = reusableTemplateConnectionInfo.getReusableTemplateInputPortName();
        // find the process group instance in the 'reusable_templates' group in NiFi for this input port
        Optional<ProcessGroupDTO> processGroupDTO = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getName().equals(inputName) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).flatMap(connectionDTO -> reusableTemplateFlow.getFlow().getProcessGroups().stream().map(processGroupEntity -> processGroupEntity.getComponent()).filter(groupDTO -> groupDTO.getId().equals(connectionDTO.getDestination().getGroupId()))).findFirst();
        if (processGroupDTO.isPresent()) {
            // walk the output ports to find any other connecting templates
            List<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new ArrayList<>();
            List<ConnectionDTO> outputPortConnections = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getSource().getGroupId().equals(processGroupDTO.get().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.OUTPUT_PORT)).collect(Collectors.toList());
            for (ConnectionDTO outputConnection : outputPortConnections) {
                // walk these and get their templates
                // first get the connection metadata info needed
                // 1 find the reusable template input port for this connected template
                Optional<ConnectionDTO> inputPortToProcessGroupConnection = reusableTemplateFlow.getFlow().getConnections().stream().map(connectionEntity -> connectionEntity.getComponent()).filter(connectionDTO -> connectionDTO.getDestination().getId().equals(outputConnection.getDestination().getId()) && connectionDTO.getSource().getType().equals(NifiConstants.INPUT_PORT)).findFirst();
                if (inputPortToProcessGroupConnection.isPresent()) {
                    ReusableTemplateConnectionInfo connectionInfo = new ReusableTemplateConnectionInfo();
                    connectionInfo.setFeedOutputPortName(outputConnection.getSource().getName());
                    connectionInfo.setReusableTemplateInputPortName(inputPortToProcessGroupConnection.get().getSource().getName());
                    connectionInfo.setInputPortDisplayName(inputPortToProcessGroupConnection.get().getSource().getName());
                    String processGroupName = reusableTemplateFlow.getFlow().getProcessGroups().stream().filter(processGroupEntity -> processGroupEntity.getComponent().getId().equals(inputPortToProcessGroupConnection.get().getDestination().getGroupId())).map(processGroupEntity -> processGroupEntity.getComponent().getName()).findFirst().orElse(null);
                    connectionInfo.setReusableTemplateProcessGroupName(processGroupName);
                    outputPortConnectionMetadata.add(connectionInfo);
                    // recursively walk these flows and gather other output port connections
                    connectingOutputPortConnectionMetadata.add(connectionInfo);
                }
            // also add in the process group if it doesnt connect
            }
            if (!outputPortConnectionMetadata.isEmpty()) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, connectingOutputPortConnectionMetadata, outputPortConnectionMetadata, reusableTemplateFlow);
            }
        }
        // find the template that has the input port name
        Map<String, String> map = nifiRestClient.getTemplatesAsXmlMatchingInputPortName(inputName, reusableTemplateConnectionInfo.getReusableTemplateProcessGroupName());
        if (map != null && !map.isEmpty()) {
            for (Map.Entry<String, String> entry : map.entrySet()) {
                String portTemplateId = entry.getKey();
                if (!connectedTemplateIds.contains(portTemplateId)) {
                    connectedTemplateIds.add(portTemplateId);
                    connectingReusableTemplates.add(entry.getValue());
                }
            }
        }
    }
}

Example 9 with ProcessGroupFlowDTO

use of org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO in project kylo by Teradata.

the class ImportReusableTemplate method rollbackCreatedItems.

/**
 * Rollback and delete any created items
 * restore any items that were deleted
 */
private void rollbackCreatedItems() {
    getItemsCreated().getCreatedRemoteInputPortConnections().stream().forEach(connection -> nifiRestClient.deleteConnection(connection, false));
    getItemsCreated().getCreatedRemoteInputPorts().stream().forEach(portDTO -> nifiRestClient.getNiFiRestClient().ports().deleteInputPort(portDTO.getId()));
    Map<String, String> oldToNewPortIdMap = new HashMap<>();
    getItemsCreated().getDeletedRemoteInputPorts().stream().forEach(portDTO -> {
        PortDTO createdPort = nifiRestClient.getNiFiRestClient().processGroups().createInputPort(portDTO.getParentGroupId(), portDTO);
        oldToNewPortIdMap.put(portDTO.getId(), createdPort.getId());
    });
    // find matching connection
    Map<String, String> newReusableTemplatePortNameToId = new HashMap<>();
    String reusableTemplateProcessGroupId = templateConnectionUtil.getReusableTemplateProcessGroupId();
    if (reusableTemplateProcessGroupId != null) {
        ProcessGroupFlowDTO reusableTemplateFlow = nifiRestClient.getNiFiRestClient().processGroups().flow(reusableTemplateProcessGroupId);
        String templateProcessGroupId = reusableTemplateFlow.getFlow().getProcessGroups().stream().filter(e -> e.getComponent().getName().equalsIgnoreCase(this.importTemplate.getTemplateName())).map(e -> e.getComponent().getId()).findFirst().orElse(null);
        if (templateProcessGroupId != null) {
            reusableTemplateFlow.getFlow().getConnections().stream().filter(connectionEntity -> connectionEntity.getComponent().getDestination().getGroupId().equalsIgnoreCase(templateProcessGroupId) && connectionEntity.getComponent().getSource().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT)).forEach(connectionEntity -> {
                newReusableTemplatePortNameToId.put(connectionEntity.getComponent().getSource().getName(), connectionEntity.getComponent().getSource().getId());
            });
        }
    }
    String rootProcessGroupId = templateConnectionUtil.getRootProcessGroup().getId();
    getItemsCreated().getDeletedRemoteInputPortConnections().stream().forEach(connectionDTO -> {
        String newId = oldToNewPortIdMap.get(connectionDTO.getSource().getId());
        connectionDTO.getSource().setId(newId);
        String newDestId = newReusableTemplatePortNameToId.get(connectionDTO.getSource().getName());
        connectionDTO.getDestination().setId(newDestId);
        ConnectionDTO restoredConnection = nifiRestClient.getNiFiRestClient().processGroups().createConnection(rootProcessGroupId, connectionDTO.getSource(), connectionDTO.getDestination());
    });
}

Example 10 with ProcessGroupFlowDTO

use of org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        if (template.usesReusableTemplate()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if we are clustered
            NiFiClusterSummary clusterSummary = nifiRestClient.getNiFiRestClient().clusterSummary();
            if (clusterSummary.getClustered()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (Exception e) {
            throw new UnsupportedOperationException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", zipFile);
    } else {
        throw new UnsupportedOperationException("Unable to find Template for " + templateId);
    }
}