Examples with NifiClientRuntimeException com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException used on opensource projects

Example 6 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}

Example 7 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class CreateFeedBuilder method rollback.

public ProcessGroupDTO rollback() throws FeedRollbackException {
    if (newProcessGroup != null) {
        try {
            removeProcessGroup(newProcessGroup.getProcessGroupEntity());
        } catch (NifiClientRuntimeException e) {
            log.error("Unable to delete the ProcessGroup on rollback {} ", e.getMessage());
        }
    }
    String parentGroupId = newProcessGroup != null ? newProcessGroup.getProcessGroupEntity().getParentGroupId() : (previousFeedProcessGroup != null ? previousFeedProcessGroup.getParentGroupId() : null);
    try {
        if (StringUtils.isNotBlank(parentGroupId)) {
            ProcessGroupDTO feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
            // rename this group to be something else if for some reason we were not able to delete it
            if (feedGroup != null) {
                feedGroup.setName(feedGroup.getName() + ".rollback - " + new Date().getTime());
                restClient.updateProcessGroup(feedGroup);
                feedGroup = restClient.getProcessGroupByName(parentGroupId, feedName);
            }
            // there shouldn't be as we should have deleted it above
            if (feedGroup == null) {
                if (previousFeedProcessGroup != null) {
                    ProcessGroupDTO entity = restClient.getProcessGroup(previousFeedProcessGroup.getId(), false, false);
                    if (entity != null) {
                        entity.setName(feedName);
                        entity = restClient.updateProcessGroup(entity);
                        ProcessGroupDTO categoryGroup = restClient.getProcessGroup(entity.getParentGroupId(), false, false);
                        updatePortConnectionsForProcessGroup(entity, categoryGroup);
                        // disable all inputs
                        restClient.disableInputProcessors(entity.getId());
                        // mark everything else as running
                        restClient.markProcessorGroupAsRunning(entity);
                        if (hasConnectionPorts()) {
                            templateCreationHelper.markConnectionPortsAsRunning(entity);
                        }
                        // Set the state correctly for the inputs
                        if (enabled) {
                            restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.RUNNING);
                        } else {
                            restClient.setInputProcessorState(entity.getId(), inputProcessorType, NifiProcessUtil.PROCESS_STATE.STOPPED);
                        }
                        return entity;
                    }
                }
            }
        }
    } catch (Exception e) {
        throw new FeedRollbackException("Unable to rollback feed [" + feedName + "] with Parent Group Id of [" + parentGroupId + "] " + e.getMessage(), e);
    }
    return null;
}

Example 8 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class CreateFeedBuilder method connectFeedToReusableTemplatexx.

private void connectFeedToReusableTemplatexx(String feedGroupId, String feedCategoryId) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    for (InputOutputPort port : inputOutputPorts) {
        stopwatch.start();
        restClient.connectFeedToGlobalTemplate(feedGroupId, port.getOutputPortName(), feedCategoryId, reusableTemplateCategoryGroupId, port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 9 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class AbstractImportTemplateRoutine method removeTemporaryProcessGroup.

/**
 * Cleanup and remove the temporary process group associated with this import
 *
 * This should be called after the import to cleanup NiFi
 */
public void removeTemporaryProcessGroup() {
    UploadProgressMessage statusMessage = uploadProgressService.addUploadStatus(importTemplate.getImportOptions().getUploadKey(), "Cleaning up the temporary process group in NiFi");
    String processGroupName = importTemplate.getTemplateResults().getProcessGroupEntity().getName();
    String processGroupId = importTemplate.getTemplateResults().getProcessGroupEntity().getId();
    String parentProcessGroupId = importTemplate.getTemplateResults().getProcessGroupEntity().getParentGroupId();
    log.info("About to cleanup the temporary process group {} ({})", processGroupName, processGroupId);
    try {
        nifiRestClient.removeProcessGroup(processGroupId, parentProcessGroupId);
        log.info("Successfully cleaned up Nifi and deleted the process group {} ", importTemplate.getTemplateResults().getProcessGroupEntity().getName());
        statusMessage.update("Removed the temporary process group " + processGroupName + " in NiFi", true);
    } catch (NifiClientRuntimeException e) {
        log.error("error attempting to cleanup and remove the temporary process group (" + processGroupId + " during the import of template " + importTemplate.getTemplateName());
        importTemplate.getTemplateResults().addError(NifiError.SEVERITY.WARN, "Issues found in cleaning up the template import: " + importTemplate.getTemplateName() + ".  The Process Group : " + processGroupName + " (" + processGroupId + ")" + " may need to be manually cleaned up in NiFi ", "");
        statusMessage.update("Error cleaning up NiFi. The Process Group : " + processGroupName + " (" + processGroupId + ")", false);
    }
}

Example 10 with NifiClientRuntimeException

use of com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException in project kylo by Teradata.

the class ImportReusableTemplate method cleanup.

public void cleanup() {
    if (newTemplateInstance != null) {
        String templateName = importTemplate.getTemplateName();
        VersionedProcessGroup versionedProcessGroup = newTemplateInstance.getVersionedProcessGroup();
        if (versionedProcessGroup != null) {
            // ensure we have contents in the versionedProcess group
            if (versionedProcessGroup.getVersionedProcessGroup().getContents() == null) {
                FlowDTO flowDTO = nifiRestClient.getNiFiRestClient().processGroups().flow(versionedProcessGroup.getVersionedProcessGroup().getId()).getFlow();
                if (flowDTO != null) {
                    versionedProcessGroup.getVersionedProcessGroup().setContents(NifiFlowUtil.flowToFlowSnippet(flowDTO));
                }
            }
            // importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup()
            uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "The Reusable template " + templateName + " is valid.  Attempting to clean up and remove the previous instance ", true, true);
            // if the versioned group doesnt have anything in queue delete it
            Optional<ProcessGroupStatusDTO> status = nifiRestClient.getNiFiRestClient().processGroups().getStatus(versionedProcessGroup.getVersionedProcessGroup().getId());
            if (!status.isPresent() || (status.isPresent() && status.get().getAggregateSnapshot().getFlowFilesQueued() > 0)) {
                uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Warning! The previous version of this template '" + templateName + "', now located in NiFi under the process group '" + versionedProcessGroup.getVersionedProcessGroupName() + "' still has items in queue.  Unable to delete it. ", true, true);
                // if the versioned group still had items in queue then wire it back up to the newly created group
                // ?? are they still wired up to the output ports
                log.info("Unable to delete versioned group '{}'  Items are still in queue", versionedProcessGroup.getVersionedProcessGroupName());
                // stop the inputs and mark the rest as running
                versionedProcessGroup.getVersionedProcessGroup().getContents().getInputPorts().stream().forEach(portDTO -> {
                    nifiRestClient.stopInputPort(versionedProcessGroup.getVersionedProcessGroup().getId(), portDTO.getId());
                });
                // start the output ports
                versionedProcessGroup.getVersionedProcessGroup().getContents().getOutputPorts().stream().forEach(portDTO -> {
                    nifiRestClient.startOutputPort(versionedProcessGroup.getVersionedProcessGroup().getId(), portDTO.getId());
                });
            } else {
                // delete the versioned group
                try {
                    nifiRestClient.removeProcessGroup(importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup().getId(), importTemplate.getTemplateResults().getVersionedProcessGroup().getVersionedProcessGroup().getParentGroupId());
                } catch (NifiClientRuntimeException e) {
                    log.error("Template has sucessfully imported, but Kylo is unable to remove versioned process group '{}' with id '{}'.  You will need to go into NiFi and manually remove it.", versionedProcessGroup.getVersionedProcessGroupName(), versionedProcessGroup.getVersionedProcessGroup().getId());
                    uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Warning! Unable to remove the previous version of this template, process group: '" + versionedProcessGroup.getVersionedProcessGroupName() + "' with id of " + versionedProcessGroup.getVersionedProcessGroup().getId() + ".  You will need to manually delete it from NiFi. ", true, false);
                }
            }
        }
    }
}