Search in sources :

Example 21 with NifiComponentNotFoundException

use of com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException in project kylo by Teradata.

the class TemplateConnectionUtil method connectFeedToReusableTemplate.

public void connectFeedToReusableTemplate(ProcessGroupDTO feedProcessGroup, ProcessGroupDTO categoryProcessGroup, List<InputOutputPort> inputOutputPorts) throws NifiComponentNotFoundException {
    Stopwatch stopwatch = Stopwatch.createStarted();
    String categoryProcessGroupId = categoryProcessGroup.getId();
    String categoryParentGroupId = categoryProcessGroup.getParentGroupId();
    String categoryProcessGroupName = categoryProcessGroup.getName();
    String feedProcessGroupId = feedProcessGroup.getId();
    String feedProcessGroupName = feedProcessGroup.getName();
    ProcessGroupDTO reusableTemplateCategory = niFiObjectCache.getReusableTemplateCategoryProcessGroup();
    if (reusableTemplateCategory == null) {
        throw new NifiClientRuntimeException("Unable to find the Reusable Template Group. Please ensure NiFi has the 'reusable_templates' processgroup and appropriate reusable flow for this feed." + " You may need to import the base reusable template for this feed.");
    }
    String reusableTemplateCategoryGroupId = reusableTemplateCategory.getId();
    stopwatch.stop();
    log.debug("Time to get reusableTemplateCategory: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
    stopwatch.reset();
    Stopwatch totalStopWatch = Stopwatch.createUnstarted();
    for (InputOutputPort port : inputOutputPorts) {
        totalStopWatch.start();
        stopwatch.start();
        PortDTO reusableTemplatePort = niFiObjectCache.getReusableTemplateInputPort(port.getInputPortName());
        stopwatch.stop();
        log.debug("Time to get reusableTemplate inputPort {} : {} ", port.getInputPortName(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
        stopwatch.reset();
        if (reusableTemplatePort != null) {
            String categoryOutputPortName = categoryProcessGroupName + " to " + port.getInputPortName();
            stopwatch.start();
            PortDTO categoryOutputPort = niFiObjectCache.getCategoryOutputPort(categoryProcessGroupId, categoryOutputPortName);
            if (categoryOutputPort != null) {
                // ensure it exists
                try {
                    categoryOutputPort = restClient.getNiFiRestClient().ports().getOutputPort(categoryOutputPort.getId());
                } catch (Exception e) {
                    categoryOutputPort = null;
                }
            }
            stopwatch.stop();
            log.debug("Time to get categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (categoryOutputPort == null) {
                stopwatch.start();
                // create it
                PortDTO portDTO = new PortDTO();
                portDTO.setParentGroupId(categoryProcessGroupId);
                portDTO.setName(categoryOutputPortName);
                categoryOutputPort = restClient.getNiFiRestClient().processGroups().createOutputPort(categoryProcessGroupId, portDTO);
                niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                stopwatch.stop();
                log.debug("Time to create categoryOutputPort {} : {} ", categoryOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
            }
            stopwatch.start();
            Set<PortDTO> feedOutputPorts = feedProcessGroup.getContents().getOutputPorts();
            String feedOutputPortName = port.getOutputPortName();
            if (feedOutputPorts == null || feedOutputPorts.isEmpty()) {
                feedOutputPorts = restClient.getNiFiRestClient().processGroups().getOutputPorts(feedProcessGroup.getId());
            }
            PortDTO feedOutputPort = NifiConnectionUtil.findPortMatchingName(feedOutputPorts, feedOutputPortName);
            stopwatch.stop();
            log.debug("Time to create feedOutputPort {} : {} ", feedOutputPortName, stopwatch.elapsed(TimeUnit.MILLISECONDS));
            stopwatch.reset();
            if (feedOutputPort != null) {
                stopwatch.start();
                // make the connection on the category from feed to category
                ConnectionDTO feedOutputToCategoryOutputConnection = niFiObjectCache.getConnection(categoryProcessGroupId, feedOutputPort.getId(), categoryOutputPort.getId());
                stopwatch.stop();
                log.debug("Time to get feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                if (feedOutputToCategoryOutputConnection == null) {
                    stopwatch.start();
                    // CONNECT FEED OUTPUT PORT TO THE Category output port
                    ConnectableDTO source = new ConnectableDTO();
                    source.setGroupId(feedProcessGroupId);
                    source.setId(feedOutputPort.getId());
                    source.setName(feedProcessGroupName);
                    source.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO dest = new ConnectableDTO();
                    dest.setGroupId(categoryProcessGroupId);
                    dest.setName(categoryOutputPort.getName());
                    dest.setId(categoryOutputPort.getId());
                    dest.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    // ensure the port exists
                    niFiObjectCache.addCategoryOutputPort(categoryProcessGroupId, categoryOutputPort);
                    feedOutputToCategoryOutputConnection = restClient.createConnection(categoryProcessGroupId, source, dest);
                    niFiObjectCache.addConnection(categoryProcessGroupId, feedOutputToCategoryOutputConnection);
                    nifiFlowCache.addConnectionToCache(feedOutputToCategoryOutputConnection);
                    stopwatch.stop();
                    log.debug("Time to create feedOutputToCategoryOutputConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
                stopwatch.start();
                // connection made on parent (root) to reusable template
                ConnectionDTO categoryToReusableTemplateConnection = niFiObjectCache.getConnection(categoryProcessGroup.getParentGroupId(), categoryOutputPort.getId(), reusableTemplatePort.getId());
                stopwatch.stop();
                log.debug("Time to get categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                stopwatch.reset();
                // Now connect the category ProcessGroup to the global template
                if (categoryToReusableTemplateConnection == null) {
                    stopwatch.start();
                    ConnectableDTO categorySource = new ConnectableDTO();
                    categorySource.setGroupId(categoryProcessGroupId);
                    categorySource.setId(categoryOutputPort.getId());
                    categorySource.setName(categoryOutputPortName);
                    categorySource.setType(NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name());
                    ConnectableDTO categoryToGlobalTemplate = new ConnectableDTO();
                    categoryToGlobalTemplate.setGroupId(reusableTemplateCategoryGroupId);
                    categoryToGlobalTemplate.setId(reusableTemplatePort.getId());
                    categoryToGlobalTemplate.setName(reusableTemplatePort.getName());
                    categoryToGlobalTemplate.setType(NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name());
                    categoryToReusableTemplateConnection = restClient.createConnection(categoryParentGroupId, categorySource, categoryToGlobalTemplate);
                    niFiObjectCache.addConnection(categoryParentGroupId, categoryToReusableTemplateConnection);
                    nifiFlowCache.addConnectionToCache(categoryToReusableTemplateConnection);
                    stopwatch.stop();
                    log.debug("Time to create categoryToReusableTemplateConnection: {} ", stopwatch.elapsed(TimeUnit.MILLISECONDS));
                    stopwatch.reset();
                }
            }
        }
        totalStopWatch.stop();
        log.debug("Time to connect feed to {} port. ElapsedTime: {} ", port.getInputPortName(), totalStopWatch.elapsed(TimeUnit.MILLISECONDS));
        totalStopWatch.reset();
    }
}
Also used : PortDTO(org.apache.nifi.web.api.dto.PortDTO) ConnectionDTO(org.apache.nifi.web.api.dto.ConnectionDTO) Stopwatch(com.google.common.base.Stopwatch) InputOutputPort(com.thinkbiganalytics.nifi.feedmgr.InputOutputPort) ProcessGroupDTO(org.apache.nifi.web.api.dto.ProcessGroupDTO) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) ConnectableDTO(org.apache.nifi.web.api.dto.ConnectableDTO) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException)

Example 22 with NifiComponentNotFoundException

use of com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException in project kylo by Teradata.

the class RegisteredTemplateService method getRegisteredTemplateForUpdate.

/**
 * Return a registered template object that is populated for use with updating in Kylo
 *
 * @param registeredTemplateRequest the request to get a registered template
 * @return a RegisteredTemplate object mapping either one already defined in Kylo, or a new template that maps to one in NiFi
 */
public RegisteredTemplate getRegisteredTemplateForUpdate(RegisteredTemplateRequest registeredTemplateRequest) {
    if (registeredTemplateRequest.isTemplateEdit()) {
        this.accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.EDIT_TEMPLATES);
    }
    RegisteredTemplate registeredTemplate = null;
    // attempt to find the template as a Service
    RegisteredTemplateRequest serviceLevelRequest = new RegisteredTemplateRequest(registeredTemplateRequest);
    // editing a feed will run as a service account
    serviceLevelRequest.setFeedEdit(true);
    RegisteredTemplate template = findRegisteredTemplate(serviceLevelRequest);
    boolean canEdit = true;
    if (template != null && StringUtils.isNotBlank(template.getId()) && registeredTemplateRequest.isTemplateEdit()) {
        canEdit = checkTemplatePermission(template.getId(), TemplateAccessControl.EDIT_TEMPLATE);
    }
    if (canEdit) {
        registeredTemplate = template;
        if (registeredTemplate == null) {
            registeredTemplate = nifiTemplateToRegisteredTemplate(registeredTemplateRequest.getNifiTemplateId());
        }
        if (registeredTemplate == null) {
        // throw exception
        } else {
            if (StringUtils.isBlank(registeredTemplate.getId()) && template != null && StringUtils.isNotBlank(template.getId())) {
                registeredTemplate.setId(template.getId());
            }
            Set<PortDTO> ports = null;
            // fetch ports for this template
            try {
                if (registeredTemplate.getNifiTemplate() != null) {
                    ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplate());
                } else {
                    ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplateId());
                }
            } catch (NifiComponentNotFoundException notFoundException) {
                syncNiFiTemplateId(registeredTemplate);
                ports = nifiRestClient.getPortsForTemplate(registeredTemplate.getNifiTemplateId());
            }
            if (ports == null) {
                ports = new HashSet<>();
            }
            List<PortDTO> outputPorts = ports.stream().filter(portDTO -> portDTO != null && NifiConstants.NIFI_PORT_TYPE.OUTPUT_PORT.name().equalsIgnoreCase(portDTO.getType())).collect(Collectors.toList());
            List<PortDTO> inputPorts = ports.stream().filter(portDTO -> portDTO != null && NifiConstants.NIFI_PORT_TYPE.INPUT_PORT.name().equalsIgnoreCase(portDTO.getType())).collect(Collectors.toList());
            registeredTemplate.setReusableTemplate(inputPorts != null && !inputPorts.isEmpty());
            List<ReusableTemplateConnectionInfo> reusableTemplateConnectionInfos = registeredTemplate.getReusableTemplateConnections();
            List<ReusableTemplateConnectionInfo> updatedConnectionInfo = new ArrayList<>();
            for (final PortDTO port : outputPorts) {
                ReusableTemplateConnectionInfo reusableTemplateConnectionInfo = null;
                if (reusableTemplateConnectionInfos != null && !reusableTemplateConnectionInfos.isEmpty()) {
                    reusableTemplateConnectionInfo = Iterables.tryFind(reusableTemplateConnectionInfos, reusableTemplateConnectionInfo1 -> reusableTemplateConnectionInfo1.getFeedOutputPortName().equalsIgnoreCase(port.getName())).orNull();
                }
                if (reusableTemplateConnectionInfo == null) {
                    reusableTemplateConnectionInfo = new ReusableTemplateConnectionInfo();
                    reusableTemplateConnectionInfo.setFeedOutputPortName(port.getName());
                }
                updatedConnectionInfo.add(reusableTemplateConnectionInfo);
            }
            registeredTemplate.setReusableTemplateConnections(updatedConnectionInfo);
            registeredTemplate.initializeProcessors();
            ensureRegisteredTemplateInputProcessors(registeredTemplate);
        }
    }
    return registeredTemplate;
}
Also used : NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) Action(com.thinkbiganalytics.security.action.Action) IntStream(java.util.stream.IntStream) Iterables(com.google.common.collect.Iterables) TEMPLATE_TRANSFORMATION_TYPE(com.thinkbiganalytics.feedmgr.service.template.TemplateModelTransform.TEMPLATE_TRANSFORMATION_TYPE) LoggerFactory(org.slf4j.LoggerFactory) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo) HashMap(java.util.HashMap) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) NifiFeedConstants(com.thinkbiganalytics.nifi.rest.support.NifiFeedConstants) StringUtils(org.apache.commons.lang3.StringUtils) RegisteredTemplateRequest(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) NiFiRemoteProcessGroup(com.thinkbiganalytics.nifi.rest.model.NiFiRemoteProcessGroup) TemplateDTO(org.apache.nifi.web.api.dto.TemplateDTO) TemplateAccessControl(com.thinkbiganalytics.metadata.api.template.security.TemplateAccessControl) NifiPropertyUtil(com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil) NifiTemplateUtil(com.thinkbiganalytics.nifi.rest.support.NifiTemplateUtil) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) FeedManagerTemplateProvider(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplateProvider) NifiConstants(com.thinkbiganalytics.nifi.rest.support.NifiConstants) Logger(org.slf4j.Logger) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) Set(java.util.Set) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) FeedManagerTemplate(com.thinkbiganalytics.metadata.api.template.FeedManagerTemplate) PortDTO(org.apache.nifi.web.api.dto.PortDTO) List(java.util.List) Principal(java.security.Principal) ProcessorDTO(org.apache.nifi.web.api.dto.ProcessorDTO) AccessControlException(java.security.AccessControlException) Collections(java.util.Collections) LegacyNifiRestClient(com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient) PortDTO(org.apache.nifi.web.api.dto.PortDTO) ArrayList(java.util.ArrayList) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) RegisteredTemplateRequest(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplateRequest) ReusableTemplateConnectionInfo(com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo)

Aggregations

NifiComponentNotFoundException (com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException)22 Nonnull (javax.annotation.Nonnull)10 NotFoundException (javax.ws.rs.NotFoundException)10 ProcessGroupDTO (org.apache.nifi.web.api.dto.ProcessGroupDTO)10 ConnectionDTO (org.apache.nifi.web.api.dto.ConnectionDTO)6 RevisionDTO (org.apache.nifi.web.api.dto.RevisionDTO)6 HashMap (java.util.HashMap)5 RemoteProcessGroupDTO (org.apache.nifi.web.api.dto.RemoteProcessGroupDTO)5 PortEntity (org.apache.nifi.web.api.entity.PortEntity)5 NifiClientRuntimeException (com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException)4 NifiVisitableProcessGroup (com.thinkbiganalytics.nifi.rest.model.visitor.NifiVisitableProcessGroup)4 LegacyNifiRestClient (com.thinkbiganalytics.nifi.rest.client.LegacyNifiRestClient)3 NifiProperty (com.thinkbiganalytics.nifi.rest.model.NifiProperty)3 NifiConstants (com.thinkbiganalytics.nifi.rest.support.NifiConstants)3 ArrayList (java.util.ArrayList)3 Collections (java.util.Collections)3 Iterables (com.google.common.collect.Iterables)2 ReusableTemplateConnectionInfo (com.thinkbiganalytics.feedmgr.rest.model.ReusableTemplateConnectionInfo)2 NifiPropertyUtil (com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil)2 HashSet (java.util.HashSet)2