Examples with NifiClientRuntimeException com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException used on opensource projects

Example 11 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class CreateFeedBuilder method build.

/**
 * Build the NiFi flow instance
 *
 * @return an object indicating if the feed flow was successfully built or not
 */
public NifiProcessGroup build() throws FeedCreationException {
    try {
        log.info("Creating the feed {}.{} ", category, feedName);
        newProcessGroup = null;
        Stopwatch totalTime = Stopwatch.createStarted();
        Stopwatch eventTime = Stopwatch.createStarted();
        TemplateDTO template = getTemplate();
        if (template != null) {
            log.debug("Time to get Template {}.  ElapsedTime: {} ms", template.getName(), eventTime(eventTime));
            // create the encompassing process group
            eventTime.start();
            ProcessGroupDTO feedProcessGroup = createProcessGroupForFeed();
            log.debug("Time to create process group.  ElapsedTime: {} ms", eventTime(eventTime));
            if (feedProcessGroup != null) {
                String processGroupId = feedProcessGroup.getId();
                // snapshot the existing controller services
                eventTime.start();
                templateCreationHelper.snapshotControllerServiceReferences();
                log.debug("Time to snapshotControllerServices.  ElapsedTime: {} ms", eventTime(eventTime));
                // create the flow from the template
                eventTime.start();
                TemplateInstance instance = templateCreationHelper.instantiateFlowFromTemplate(processGroupId, templateId);
                FlowSnippetDTO feedInstance = instance.getFlowSnippetDTO();
                feedProcessGroup.setContents(feedInstance);
                log.debug("Time to instantiateFlowFromTemplate.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                String feedCategoryId = feedProcessGroup.getParentGroupId();
                ProcessGroupDTO categoryGroup = this.categoryGroup;
                if (categoryGroup == null) {
                    categoryGroup = this.categoryGroup = restClient.getProcessGroup(feedCategoryId, false, false);
                }
                // update the group with this template?
                updatePortConnectionsForProcessGroup(feedProcessGroup, categoryGroup);
                log.debug("Time to updatePortConnectionsForProcessGroup.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                // mark the new services that were created as a result of creating the new flow from the template
                templateCreationHelper.identifyNewlyCreatedControllerServiceReferences(instance);
                log.debug("Time to identifyNewlyCreatedControllerServiceReferences.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                // match the properties incoming to the defined properties
                updateProcessGroupProperties(processGroupId, feedProcessGroup.getName());
                log.debug("Time to updateProcessGroupProperties.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                // Fetch the Feed Group now that it has the flow in it
                ProcessGroupDTO entity = restClient.getProcessGroup(processGroupId, true, true);
                log.debug("Time to getProcessGroup.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                ProcessorDTO input = fetchInputProcessorForProcessGroup(entity);
                ProcessorDTO cleanupProcessor = NifiProcessUtil.findFirstProcessorsByType(NifiProcessUtil.getInputProcessors(entity), "com.thinkbiganalytics.nifi.v2.metadata.TriggerCleanup");
                List<ProcessorDTO> nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                log.debug("Time to fetchInputProcessorForProcessGroup.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                List<NifiProperty> updatedControllerServiceProperties = new ArrayList<>();
                // update any references to the controller services and try to assign the value to an enabled service if it is not already
                if (input != null) {
                    updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Lists.newArrayList(input), instance));
                }
                if (cleanupProcessor != null) {
                    updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(Collections.singletonList(cleanupProcessor), instance));
                }
                updatedControllerServiceProperties.addAll(templateCreationHelper.updateControllerServiceReferences(nonInputProcessors, instance));
                log.debug("Time to updatedControllerServiceProperties.  ElapsedTime: {} ms", eventTime(eventTime));
                eventTime.start();
                // refetch processors for updated errors
                entity = restClient.getProcessGroup(processGroupId, true, true);
                input = fetchInputProcessorForProcessGroup(entity);
                nonInputProcessors = NifiProcessUtil.getNonInputProcessors(entity);
                newProcessGroup = new NifiProcessGroup(entity, input, nonInputProcessors);
                log.debug("Time to re-fetchInputProcessorForProcessGroup.  ElapsedTime: {} ms", eventTime(eventTime));
                // Validate and if invalid Delete the process group
                if (newProcessGroup.hasFatalErrors()) {
                    eventTime.start();
                    removeProcessGroup(entity);
                    // cleanupControllerServices();
                    newProcessGroup.setSuccess(false);
                    log.debug("Time to removeProcessGroup. Errors found.  ElapsedTime: {} ms", eventTime(eventTime));
                } else {
                    eventTime.start();
                    // update the input schedule
                    updateFeedSchedule(newProcessGroup, input);
                    log.debug("Time to update feed schedule.  ElapsedTime: {} ms", eventTime(eventTime));
                    eventTime.start();
                    // just need to update for this processgroup
                    Collection<ProcessorDTO> processors = NifiProcessUtil.getProcessors(entity);
                    Collection<ConnectionDTO> connections = NifiConnectionUtil.getAllConnections(entity);
                    nifiFlowCache.updateFlowForFeed(feedMetadata, entity.getId(), processors, connections);
                    log.debug("Time to build flow graph with {} processors and {} connections.  ElapsedTime: {} ms", processors.size(), connections.size(), eventTime(eventTime));
                    /*

                        //Cache the processorIds to the respective flowIds for availability in the ProvenanceReportingTask
                        NifiVisitableProcessGroup group = nifiFlowCache.getFlowOrder(newProcessGroup.getProcessGroupEntity(), true);
                       log.debug("Time to get the flow order.  ElapsedTime: {} ms", eventTime(eventTime));

                        eventTime.start();
                        NifiFlowProcessGroup
                            flow =
                            new NifiFlowBuilder().build(
                                group);
                       log.debug("Time to build flow graph with {} processors.  ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));

                        eventTime.start();
                        nifiFlowCache.updateFlow(feedMetadata, flow);
                       log.debug("Time to update NiFiFlowCache with {} processors.  ElapsedTime: {} ms", flow.getProcessorMap().size(), eventTime(eventTime));
                        */
                    eventTime.start();
                    // disable all inputs
                    restClient.disableInputProcessors(newProcessGroup.getProcessGroupEntity().getId());
                    log.debug("Time to disableInputProcessors.  ElapsedTime: {} ms", eventTime(eventTime));
                    eventTime.start();
                    // mark everything else as running
                    templateCreationHelper.markProcessorsAsRunning(newProcessGroup);
                    log.debug("Time to markNonInputsAsRunning.  ElapsedTime: {} ms", eventTime(eventTime));
                    // if desired start the input processor
                    if (input != null) {
                        eventTime.start();
                        if (enabled) {
                            markInputAsRunning(newProcessGroup, input);
                            // /make the input/output ports in the category group as running
                            if (hasConnectionPorts()) {
                                templateCreationHelper.markConnectionPortsAsRunning(entity);
                            }
                        } else {
                            // /make the input/output ports in the category group as running
                            if (hasConnectionPorts()) {
                                templateCreationHelper.markConnectionPortsAsRunning(entity);
                            }
                            markInputAsStopped(newProcessGroup, input);
                        }
                        log.debug("Time to mark input as {}.  ElapsedTime: {} ms", (enabled ? "Running" : "Stopped"), eventTime(eventTime));
                    }
                    if (newProcessGroup.hasFatalErrors()) {
                        eventTime.start();
                        rollback();
                        newProcessGroup.setRolledBack(true);
                        // cleanupControllerServices();
                        newProcessGroup.setSuccess(false);
                        log.debug("Time to rollback on Fatal Errors.  ElapsedTime: {} ms", eventTime(eventTime));
                    }
                    List<NifiError> templateCreationErrors = templateCreationHelper.getErrors();
                    if (templateCreationErrors != null) {
                        errors.addAll(templateCreationErrors);
                    }
                    // add any global errors to the object
                    if (errors != null && !errors.isEmpty()) {
                        for (NifiError error : errors) {
                            newProcessGroup.addError(error);
                            if (error.isFatal()) {
                                newProcessGroup.setSuccess(false);
                                if (!newProcessGroup.isRolledBack()) {
                                    rollback();
                                    newProcessGroup.setRolledBack(true);
                                }
                            }
                        }
                    }
                }
                eventTime.start();
                templateCreationHelper.cleanupControllerServices();
                // fix the feed metadata controller service references
                updateFeedMetadataControllerServiceReferences(updatedControllerServiceProperties);
                log.debug("Time cleanup controller services.  ElapsedTime: {} ms", eventTime(eventTime));
                // align items
                if (this.autoAlign) {
                    eventTime.start();
                    log.info("Aligning Feed flows in NiFi ");
                    AlignProcessGroupComponents alignProcessGroupComponents = new AlignProcessGroupComponents(restClient.getNiFiRestClient(), entity.getParentGroupId());
                    alignProcessGroupComponents.autoLayout();
                    // fetch the parent to get that id to align
                    if (newCategory) {
                        log.info("This is the first feed created in the category {}.  Aligning the categories. ", feedMetadata.getCategory().getSystemName());
                        new AlignProcessGroupComponents(restClient.getNiFiRestClient(), this.categoryGroup.getParentGroupId()).autoLayout();
                    }
                    log.info("Time align feed process groups.  ElapsedTime: {} ms", eventTime(eventTime));
                } else {
                    log.info("Skipping auto alignment in NiFi. You can always manually align this category and all of its feeds by using the rest api: /v1/feedmgr/nifi/auto-align/{}", entity.getParentGroupId());
                    if (newCategory) {
                        log.info("To re align the categories: /v1/feedmgr/nifi/auto-align/{}", this.categoryGroup.getParentGroupId());
                    }
                }
            }
        } else {
            log.error("Unable to create/save the feed {}.  Unable to find a template for id {}", feedName, templateId);
            throw new FeedCreationException("Unable to create the feed [" + feedName + "]. Unable to find a template with id " + templateId);
        }
        log.info("Time save Feed flow in NiFi.  ElapsedTime: {} ms", eventTime(totalTime));
        return newProcessGroup;
    } catch (NifiClientRuntimeException e) {
        throw new FeedCreationException("Unable to create the feed [" + feedName + "]. " + e.getMessage(), e);
    }
}

Example 12 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class TemplateConnectionUtil method connectFeedToReusableTemplate.

public void connectFeedToReusableTemplate(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup, List<InputOutputPort> inputOutputPorts) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            if (categoryOutputPort != null) {
                // ensure it exists
                try {
                    categoryOutputPort = restClient.getNiFiRestClient().ports().getOutputPort(categoryOutputPort.getId());
                } catch (Exception e) {
                    categoryOutputPort = null;
                }
            }
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    // ensure the port exists
                    niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 13 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class DefaultTemplateExporter method export.

private ExportTemplate export(String templateId) {
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeSensitiveProperties(true).build());
    if (template != null) {
        List<String> connectingReusableTemplates = new ArrayList<>();
        Set<String> connectedTemplateIds = new HashSet<>();
        Set<ReusableTemplateConnectionInfo> outputPortConnectionMetadata = new HashSet<>();
        Set<RemoteProcessGroupInputPort> templateRemoteInputPorts = new HashSet<>();
        if (template.usesReusableTemplate()) {
            ProcessGroupFlowDTO reusableTemplateFlow = templateConnectionUtil.getReusableTemplateCategoryProcessGroupFlow();
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = template.getReusableTemplateConnections();
            Map<String, PortDTOWithGroupInfo> reusableTemplatePorts = templateConnectionUtil.getReusableFeedInputPorts(reusableTemplateFlow).stream().collect(Collectors.toMap(port -> port.getName(), port -> port));
            reusableTemplateConnectionInfos.stream().filter(connectionInfo -> StringUtils.isBlank(connectionInfo.getReusableTemplateProcessGroupName())).forEach(connectionInfo -> {
                PortDTOWithGroupInfo port = reusableTemplatePorts.get(connectionInfo.getReusableTemplateInputPortName());
                if (port != null) {
                    connectionInfo.setReusableTemplateProcessGroupName(port.getDestinationProcessGroupName());
                }
            });
            // Get flow information for the 'reusable_templates' process group in NiFi
            if (reusableTemplateFlow != null) {
                gatherConnectedReusableTemplates(connectingReusableTemplates, connectedTemplateIds, outputPortConnectionMetadata, reusableTemplateConnectionInfos, reusableTemplateFlow);
            }
            // Only gather remote input ports on the reusable templates if we are clustered
            NiFiClusterSummary clusterSummary = nifiRestClient.getNiFiRestClient().clusterSummary();
            if (clusterSummary.getClustered()) {
                // for all the reusable templates used gather any that have remote input ports
                reusableTemplateConnectionInfos.stream().forEach(connectionInfo -> {
                    Set<RemoteProcessGroupInputPort> remoteProcessGroupInputPorts = findReusableTemplateRemoteInputPorts(reusableTemplateFlow, connectionInfo.getReusableTemplateProcessGroupName());
                    templateRemoteInputPorts.addAll(remoteProcessGroupInputPorts);
                });
            }
        }
        String templateXml = null;
        try {
            if (template != null) {
                try {
                    templateXml = nifiRestClient.getTemplateXml(template.getNifiTemplateId());
                } catch (NifiClientRuntimeException e) {
                    TemplateDTO templateDTO = nifiRestClient.getTemplateByName(template.getTemplateName());
                    if (templateDTO != null) {
                        templateXml = nifiRestClient.getTemplateXml(templateDTO.getId());
                    }
                }
            }
        } catch (Exception e) {
            throw new UnsupportedOperationException("Unable to find Nifi Template for " + templateId);
        }
        // create a zip file with the template and xml
        byte[] zipFile = zip(template, templateXml, connectingReusableTemplates, outputPortConnectionMetadata, templateRemoteInputPorts);
        return new ExportTemplate(SystemNamingService.generateSystemName(template.getTemplateName()) + ".template.zip", zipFile);
    } else {
        throw new UnsupportedOperationException("Unable to find Template for " + templateId);
    }
}