Examples with NifiProperty com.thinkbiganalytics.nifi.rest.model.NifiProperty used on opensource projects

Example 11 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class DefaultFeedManagerTemplateService method getTemplateProperties.

/**
 * Return properties registered for a template
 *
 * @param templateId a RegisteredTemplate id
 * @return the properties registered for the template
 */
public List<NifiProperty> getTemplateProperties(String templateId) {
    List<NifiProperty> list = new ArrayList<>();
    RegisteredTemplate template = registeredTemplateService.findRegisteredTemplate(RegisteredTemplateRequest.requestByTemplateId(templateId));
    if (template != null) {
        list = template.getProperties();
    }
    return list;
}

Example 12 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class RegisteredTemplateService method mergeRegisteredTemplateProperties.

/**
 * Merge all saved properties on the RegisteredTemplate along with the properties in the NiFi template
 * The resulting object will have the {@link RegisteredTemplate#properties} updated so they are in sync with NiFi.
 *
 * @return a RegisteredTemplate that has the properties updated with those in NiFi
 */
public RegisteredTemplate mergeRegisteredTemplateProperties(RegisteredTemplate registeredTemplate, RegisteredTemplateRequest registeredTemplateRequest) {
    if (registeredTemplate != null) {
        log.info("Merging properties for template {} ({})", registeredTemplate.getTemplateName(), registeredTemplate.getId());
        List<NifiProperty> properties = null;
        int matchCount = 0;
        // get the nifi template associated with this one that is registered
        TemplateDTO templateDTO = registeredTemplate.getNifiTemplate();
        if (templateDTO == null) {
            templateDTO = ensureNifiTemplate(registeredTemplate);
        }
        if (templateDTO != null) {
            registeredTemplate.setNifiTemplate(templateDTO);
            properties = niFiTemplateCache.getTemplateProperties(templateDTO, registeredTemplateRequest.isIncludePropertyDescriptors(), registeredTemplate);
        // first attempt to match the properties by the processorid and processor name
        // NifiPropertyUtil
        // .matchAndSetPropertyByIdKey(properties, registeredTemplate.getProperties(), NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
        }
        /*   if (properties != null) {
                //match the properties to the processors by the processor name
                NifiPropertyUtil.matchAndSetPropertyByProcessorName(properties, registeredTemplate.getProperties(),
                                                                    NifiPropertyUtil.PROPERTY_MATCH_AND_UPDATE_MODE.UPDATE_ALL_PROPERTIES);
            }
            */
        if (templateDTO != null && !templateDTO.getId().equalsIgnoreCase(registeredTemplate.getNifiTemplateId())) {
            syncNiFiTemplateId(registeredTemplate);
        }
        if (properties == null) {
            properties = new ArrayList<>();
        }
        // merge with the registered properties
        RegisteredTemplate copy = new RegisteredTemplate(registeredTemplate);
        copy.setProperties(properties);
        copy.setNifiTemplate(registeredTemplate.getNifiTemplate());
        registeredTemplate = copy;
    } else {
        log.info("Unable to merge Registered Template.  It is null");
    }
    return registeredTemplate;
}

Example 13 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class RegisteredTemplateService method nifiTemplateToRegisteredTemplate.

/**
 * this will return a RegisteredTemplate object for a given NiFi template Id.
 * This is to be used for new templates that are going to be registered with Kylo.
 * Callers of this method should ensure that a template of this id is not already registered
 *
 * @param nifiTemplateId the nifi template identifier
 * @return a RegisteredTemplate object of null if not found in NiFi
 */
private RegisteredTemplate nifiTemplateToRegisteredTemplate(String nifiTemplateId) {
    RegisteredTemplate registeredTemplate = null;
    List<NifiProperty> properties = new ArrayList<>();
    TemplateDTO nifiTemplate = nifiRestClient.getTemplateById(nifiTemplateId);
    if (nifiTemplate != null) {
        registeredTemplate = new RegisteredTemplate();
        registeredTemplate.setNifiTemplateId(nifiTemplateId);
        properties = niFiTemplateCache.getTemplateProperties(nifiTemplate, true, null);
        // TODO not sure if this is needed
        List<NiFiRemoteProcessGroup> remoteProcessGroups = niFiTemplateCache.getRemoteProcessGroups(nifiTemplate);
        registeredTemplate.setRemoteProcessGroups(remoteProcessGroups);
        registeredTemplate.setNifiTemplate(nifiTemplate);
        registeredTemplate.setTemplateName(nifiTemplate.getName());
        registeredTemplate.setProperties(properties);
    }
    return registeredTemplate;
}

Example 14 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class RegisteredTemplateService method ensureFeedPropertiesExistInTemplate.

/**
 * If a Template changes the Processor Names the Feed Properties will no longer be associated to the correct processors
 * This will match any feed properties to a whose processor name has changed in
 * the template to the template processor/property based upon the template processor type.
 */
private void ensureFeedPropertiesExistInTemplate(FeedMetadata feed, List<NifiProperty> templateProperties) {
    Set<String> templateProcessors = templateProperties.stream().map(property -> property.getProcessorName()).collect(Collectors.toSet());
    // Store the template Properties
    Map<String, String> templateProcessorIdProcessorNameMap = new HashMap<>();
    Map<String, String> templateProcessorTypeProcessorIdMap = new HashMap<>();
    templateProperties.stream().filter(property -> !templateProcessorIdProcessorNameMap.containsKey(property.getProcessorId())).forEach(property1 -> {
        templateProcessorIdProcessorNameMap.put(property1.getProcessorId(), property1.getProcessorName());
        templateProcessorTypeProcessorIdMap.put(property1.getProcessorType(), property1.getProcessorId());
    });
    Map<String, Map<String, NifiProperty>> templatePropertiesByProcessorIdMap = new HashMap<>();
    templateProperties.stream().forEach(property -> {
        templatePropertiesByProcessorIdMap.computeIfAbsent(property.getProcessorId(), key -> new HashMap<String, NifiProperty>()).put(property.getKey(), property);
    });
    // store the Feed Properties
    Map<String, String> processorIdProcessorTypeMap = new HashMap<>();
    feed.getProperties().stream().filter(property -> !processorIdProcessorTypeMap.containsKey(property.getProcessorId())).forEach(property1 -> {
        processorIdProcessorTypeMap.put(property1.getProcessorId(), property1.getProcessorType());
    });
    feed.getProperties().stream().filter(property -> !templateProcessors.contains(property.getProcessorName())).forEach(property -> {
        // if the property doesn't match the template but the type matches try to merge in the feed properties overwriting the template ones
        String processorType = processorIdProcessorTypeMap.get(property.getProcessorId());
        if (processorType != null) {
            String templateProcessorId = templateProcessorTypeProcessorIdMap.get(processorType);
            if (templateProcessorId != null && templateProcessorIdProcessorNameMap.containsKey(templateProcessorId)) {
                NifiProperty templateProperty = templatePropertiesByProcessorIdMap.get(templateProcessorId).get(property.getKey());
                if (templateProperty != null) {
                    // replace it from the collection with a copy
                    NifiProperty copy = new NifiProperty(templateProperty);
                    copy.setValue(property.getValue());
                    copy.setRenderType(property.getRenderType());
                    copy.setRenderOptions(property.getRenderOptions());
                    templateProperties.remove(templateProperty);
                    templateProperties.add(copy);
                }
            }
        }
    });
}

Example 15 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class ImportReusableTemplate method create.

@Override
public NifiProcessGroup create(NiFiTemplateImport niFiTemplateImport, UploadProgressMessage importStatusMessage) {
    TemplateDTO dto = niFiTemplateImport.getDto();
    String templateName = importTemplate.getTemplateName();
    String fileName = importTemplate.getFileName();
    importStatusMessage.update("Creating reusable flow instance for " + templateName);
    log.info("Creating a Reusable flow template instance in Nifi. Template: {} for file {}", templateName, fileName);
    Map<String, Object> configProperties = propertyExpressionResolver.getStaticConfigProperties();
    NifiFlowCacheReusableTemplateCreationCallback reusableTemplateCreationCallback = new NifiFlowCacheReusableTemplateCreationCallback();
    List<NifiProperty> templateProperties = importTemplate.getTemplateToImport() != null ? importTemplate.getTemplateToImport().getProperties() : Collections.emptyList();
    NifiProcessGroup newTemplateInstance = nifiRestClient.createNewTemplateInstance(dto.getId(), templateProperties, configProperties, true, reusableTemplateCreationCallback, importTemplate.getVersionIdentifier());
    if (newTemplateInstance.getVersionedProcessGroup() != null && StringUtils.isNotBlank(newTemplateInstance.getVersionedProcessGroup().getVersionedProcessGroupName())) {
        uploadProgressService.addUploadStatus(importTemplateOptions.getUploadKey(), "Versioned off previous flow with the name: " + newTemplateInstance.getVersionedProcessGroup().getVersionedProcessGroupName(), true, true);
    }
    importTemplate.setTemplateResults(newTemplateInstance);
    return newTemplateInstance;
}