Examples with RegisteredTemplate com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate used on opensource projects

Example 26 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getRegisteredTemplateProcessors.

@Override
public List<RegisteredTemplate.Processor> getRegisteredTemplateProcessors(String templateId, boolean includeReusableProcessors) {
    List<RegisteredTemplate.Processor> processorProperties = new ArrayList<>();
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(templateId).nifiTemplateId(templateId).includeAllProperties(true).build());
    if (template != null) {
        template.initializeProcessors();
        processorProperties.addAll(template.getInputProcessors());
        processorProperties.addAll(template.getNonInputProcessors());
        if (includeReusableProcessors && template.getReusableTemplateConnections() != null && !template.getReusableTemplateConnections().isEmpty()) {
            // 1 fetch ports in reusable templates
            Map<String, PortDTO> reusableTemplateInputPorts = new HashMap<>();
            Set<PortDTOWithGroupInfo> ports = getReusableFeedInputPorts();
            if (ports != null) {
                ports.stream().forEach(portDTO -> reusableTemplateInputPorts.put(portDTO.getName(), portDTO));
            }
            // match to the name
            List<String> matchingPortIds = template.getReusableTemplateConnections().stream().filter(conn -> reusableTemplateInputPorts.containsKey(conn.getReusableTemplateInputPortName())).map(reusableTemplateConnectionInfo -> reusableTemplateInputPorts.get(reusableTemplateConnectionInfo.getReusableTemplateInputPortName()).getId()).collect(Collectors.toList());
            List<RegisteredTemplate.Processor> reusableProcessors = getReusableTemplateProcessorsForInputPorts(matchingPortIds);
            processorProperties.addAll(reusableProcessors);
        }
    }
    return processorProperties;
}

Example 27 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method notifyOperationsManagerOfStreamingUpdate.

/**
 * Updates the Streaming flag on the feeds related to this template, when the template is updated and the streaming flag is changed
 *
 * @param templateName     the name of the template
 * @param previousTemplate the previous template
 */
private void notifyOperationsManagerOfStreamingUpdate(String templateName, RegisteredTemplate previousTemplate) {
    RegisteredTemplate newTemplate = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestAccessAsServiceAccountByTemplateName(templateName));
    if (newTemplate != null) {
        // notify if this is the first template update, or if we have changed the streaming flag
        boolean notifyOfStreamingChange = (previousTemplate == null || previousTemplate.isStream() != newTemplate.isStream());
        boolean notifyOfTimeChange = (previousTemplate == null || previousTemplate.getTimeBetweenStartingBatchJobs() != newTemplate.getTimeBetweenStartingBatchJobs());
        if (notifyOfStreamingChange) {
            Set<String> feedNames = newTemplate.getFeedNames();
            if (feedNames != null && !feedNames.isEmpty()) {
                metadataAccess.commit(() -> opsManagerFeedProvider.updateStreamingFlag(feedNames, newTemplate.isStream()), MetadataAccess.SERVICE);
                // Notify NiFi of the change in streaming status change
                streamingFeedJmsNotificationService.notifyNiFiOfFeedBatchStreamChange(feedNames, newTemplate.isStream());
            }
        }
        if (notifyOfTimeChange) {
            Set<String> feedNames = newTemplate.getFeedNames();
            if (feedNames != null && !feedNames.isEmpty()) {
                metadataAccess.commit(() -> opsManagerFeedProvider.updateTimeBetweenBatchJobs(feedNames, newTemplate.getTimeBetweenStartingBatchJobs()), MetadataAccess.SERVICE);
            }
        }
    }
}

Example 28 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class NifiFlowCacheImpl method applyClusterUpdates.

/**
 * if Kylo is clustered it needs to sync any updates from the other Kylo instances before proceeding
 */
public synchronized void applyClusterUpdates() {
    List<NifiFlowCacheClusterUpdateMessage> updates = nifiFlowCacheClusterManager.findUpdates();
    Set<String> templateUpdates = new HashSet<>();
    boolean needsUpdates = !updates.isEmpty();
    if (needsUpdates) {
        log.info("Kylo Cluster Update: Detected changes.  About to apply {} updates ", updates.size());
    }
    updates.stream().forEach(update -> {
        switch(update.getType()) {
            case FEED:
                NifiFlowCacheFeedUpdate feedUpdate = nifiFlowCacheClusterManager.getFeedUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Feed Change update for {}", feedUpdate.getFeedName());
                updateFlow(feedUpdate);
                break;
            case FEED2:
                NifiFlowCacheFeedUpdate2 feedUpdate2 = nifiFlowCacheClusterManager.getFeedUpdate2(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Feed Change update for {}", feedUpdate2.getFeedName());
                updateFlow(feedUpdate2);
                break;
            case CONNECTION:
                Collection<ConnectionDTO> connectionDTOS = nifiFlowCacheClusterManager.getConnectionsUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Connection list update");
                updateConnectionMap(connectionDTOS, false);
                if (connectionDTOS != null) {
                    connectionDTOS.stream().forEach(c -> {
                        niFiObjectCache.addConnection(c.getParentGroupId(), c);
                    });
                }
                break;
            case PROCESSOR:
                Collection<ProcessorDTO> processorDTOS = nifiFlowCacheClusterManager.getProcessorsUpdate(update.getMessage());
                log.info("Kylo Cluster Update:  Applying Processor list update");
                updateProcessorIdNames(processorDTOS, false);
                break;
            case TEMPLATE:
                if (!templateUpdates.contains(update.getMessage())) {
                    RegisteredTemplate template = nifiFlowCacheClusterManager.getTemplate(update.getMessage());
                    log.info("Kylo Cluster Update:  Applying Template update for {} ", template.getTemplateName());
                    updateRegisteredTemplate(template, false);
                    templateUpdates.add(update.getMessage());
                }
                break;
            default:
                break;
        }
    });
    if (needsUpdates) {
        nifiFlowCacheClusterManager.appliedUpdates(updates);
        lastUpdated = DateTime.now();
        log.info("Kylo Cluster Update: NiFi Flow File Cache is in sync. All {} updates have been applied to the cache. ", updates.size());
    }
}

Example 29 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class InMemoryFeedManagerTemplateService method postConstruct.

@PostConstruct
private void postConstruct() {
    Collection<RegisteredTemplate> templates = FileObjectPersistence.getInstance().getTemplatesFromFile();
    if (templates != null) {
        for (RegisteredTemplate t : templates) {
            String id = t.getId() == null ? t.getNifiTemplateId() : t.getId();
            registeredTemplates.put(id, t);
        }
    }
}

Example 30 with RegisteredTemplate

use of com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate in project kylo by Teradata.

the class RegisteredTemplateService method mergeTemplatePropertiesWithFeed.

public FeedMetadata mergeTemplatePropertiesWithFeed(FeedMetadata feedMetadata) {
    // gets the feed data and then gets the latest template associated with that feed and merges the properties into the feed
    RegisteredTemplate registeredTemplate = findRegisteredTemplate(new RegisteredTemplateRequest.Builder().templateId(feedMetadata.getTemplateId()).isFeedEdit(true).nifiTemplateId(feedMetadata.getTemplateId()).includeAllProperties(true).build());
    if (registeredTemplate != null) {
        feedMetadata.setTemplateId(registeredTemplate.getId());
        List<NifiProperty> templateProperties = registeredTemplate.getProperties().stream().map(nifiProperty -> new NifiProperty(nifiProperty)).collect(Collectors.toList());
        NifiPropertyUtil.matchAndSetPropertyByProcessorName(templateProperties, feedMetadata.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.FEED_DETAILS_MATCH_TEMPLATE);
        // detect template properties that dont match the feed.properties from the registeredtemplate
        ensureFeedPropertiesExistInTemplate(feedMetadata, templateProperties);
        feedMetadata.setProperties(templateProperties);
        registeredTemplate.setProperties(templateProperties);
        registeredTemplate.initializeProcessors();
        feedMetadata.setRegisteredTemplate(registeredTemplate);
    }
    return feedMetadata;
}