Examples with ProcessGroupDTO org.apache.nifi.web.api.dto.ProcessGroupDTO used on opensource projects

Example 46 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method fetchInputProcessorForProcessGroup.

private ProcessorDTO fetchInputProcessorForProcessGroup(ProcessGroupDTO entity) {
    // Find first processor by type
    final List<ProcessorDTO> inputProcessors = NifiProcessUtil.getInputProcessors(entity);
    String inputProcessorName = feedMetadata != null ? feedMetadata.getInputProcessorName() : null;
    final ProcessorDTO input = Optional.ofNullable(NifiProcessUtil.findFirstProcessorsByTypeAndName(inputProcessors, inputProcessorType, inputProcessorName)).orElseGet(() -> inputProcessors.stream().filter(processor -> !processor.getType().equals(NifiProcessUtil.CLEANUP_TYPE)).findFirst().orElse(null));
    // Update cached type
    if (input != null) {
        inputProcessorType = input.getType();
    }
    return input;
}

Example 47 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 48 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method rollback.

public ProcessGroupDTO rollback() throws FeedRollbackException {
    if (newProcessGroup != null) {
        try {
            removeProcessGroup(newProcessGroup.getProcessGroupEntity());
        } catch (NifiClientRuntimeException e) {
            log.error("Unable to delete the ProcessGroup on rollback {} ", e.getMessage());
        }
    }
    String parentGroupId = newProcessGroup != null ? newProcessGroup.getProcessGroupEntity().getParentGroupId() : (previousFeedProcessGroup != null ? previousFeedProcessGroup.getParentGroupId() : null);
    try {
        if (StringUtils.isNotBlank(parentGroupId)) {
            ProcessGroupDTO feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
            // rename this group to be something else if for some reason we were not able to delete it
            if (feedGroup != null) {
                feedGroup.setName(feedGroup.getName() + ".rollback - " + new Date().getTime());
                restClient.updateProcessGroup(feedGroup);
                feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
            }
            // there shouldn't be as we should have deleted it above
            if (feedGroup == null) {
                if (previousFeedProcessGroup != null) {
                    ProcessGroupDTO entity = restClient.getProcessGroup(previousFeedProcessGroup.getId(), false, false);
                    if (entity != null) {
                        entity.setName(feedName);
                        entity = restClient.updateProcessGroup(entity);
                        ProcessGroupDTO categoryGroup = restClient.getProcessGroup(entity.getParentGroupId(), false, false);
                        updatePortConnectionsForProcessGroup(entity, categoryGroup);
                        // disable all inputs
                        restClient.disableInputProcessors(entity.getId());
                        // mark everything else as running
                        restClient.markProcessorGroupAsRunning(entity);
                        if (hasConnectionPorts()) {
                            templateCreationHelper.markConnectionPortsAsRunning(entity);
                        }
                        // Set the state correctly for the inputs
                        if (enabled) {
                            restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.RUNNING);
                        } else {
                            restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.STOPPED);
                        }
                        return entity;
                    }
                }
            }
        }
    } catch (Exception e) {
        throw new FeedRollbackException("Unable to rollback feed [" + feedName + "] with Parent Group Id of [" + parentGroupId + "] " + e.getMessage(), e);
    }
    return null;
}

Example 49 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(String feedGroupId, String feedCategoryId) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    for (InputOutputPort port : inputOutputPorts) {
        stopwatch.start();
        restClient.connectFeedToGlobalTemplate(feedGroupId, port.getOutputPortName(), feedCategoryId, reusableTemplateCategoryGroupId, port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 50 with ProcessGroupDTO

use of org.apache.nifi.web.api.dto.ProcessGroupDTO in project kylo by Teradata.

the class TemplateConnectionUtil method getRemoteInputPortsForReusableTemplate.

public Optional<TemplateRemoteInputPortConnections> getRemoteInputPortsForReusableTemplate(ProcessGroupDTO reusableTemplateProcessGroup, String templateName) {
    String templateGroupId = reusableTemplateProcessGroup.getContents().getProcessGroups().stream().filter(g -> g.getName().equalsIgnoreCase(templateName)).map(g -> g.getId()).findFirst().orElse(null);
    if (templateGroupId != null) {
        String reusableTemplateProcessGroupId = reusableTemplateProcessGroup.getId();
        String rootProcessGroupId = this.getRootProcessGroup().getId();
        List<String> reusableTemplateInputPortIds = reusableTemplateProcessGroup.getContents().getConnections().stream().filter(conn -> conn.getDestination().getGroupId().equalsIgnoreCase(templateGroupId)).map(connectionDTO -> connectionDTO.getSource().getId()).collect(Collectors.toList());
        List<ConnectionDTO> remoteConnectionsToTemplate = getRootProcessGroupConnections().stream().filter(conn -> conn.getDestination().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && conn.getDestination().getGroupId().equalsIgnoreCase(reusableTemplateProcessGroupId) && conn.getSource().getGroupId().equalsIgnoreCase(rootProcessGroupId) && conn.getSource().getType().equalsIgnoreCase(NifiConstants.INPUT_PORT) && reusableTemplateInputPortIds.contains(conn.getDestination().getId())).collect(Collectors.toList());
        Set<String> remoteInputPorts = remoteConnectionsToTemplate.stream().map(conn -> conn.getSource().getName()).collect(Collectors.toSet());
        return Optional.of(new TemplateRemoteInputPortConnections(remoteConnectionsToTemplate, remoteInputPorts));
    }
    return Optional.empty();
}