Examples with ControllerServiceDTO org.apache.nifi.web.api.dto.ControllerServiceDTO used on opensource projects

Example 1 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class ConfigurationPropertyReplacer method replaceControllerServiceProperties.

/**
 * This will replace the Map of Properties in the DTO but not persist back to Nifi.  You need to call the rest client to persist the change
 *
 * @param controllerServiceDTO        the controller service
 * @param properties                  the properties to set
 * @param propertyDescriptorTransform transformer
 * @return {@code true} if the properties were updated, {@code false} if not
 */
public static boolean replaceControllerServiceProperties(ControllerServiceDTO controllerServiceDTO, Map<String, String> properties, NiFiPropertyDescriptorTransform propertyDescriptorTransform) {
    Set<String> changedProperties = new HashSet<>();
    if (controllerServiceDTO != null) {
        // check both Nifis Internal Key name as well as the Displayname to match the properties
        CaseInsensitiveMap propertyMap = new CaseInsensitiveMap(properties);
        Map<String, String> controllerServiceProperties = controllerServiceDTO.getProperties();
        controllerServiceProperties.entrySet().stream().filter(entry -> (propertyMap.containsKey(entry.getKey()) || (controllerServiceDTO.getDescriptors().get(entry.getKey()) != null && propertyMap.containsKey(controllerServiceDTO.getDescriptors().get(entry.getKey()).getDisplayName().toLowerCase())))).forEach(entry -> {
            boolean isSensitive = propertyDescriptorTransform.isSensitive(controllerServiceDTO.getDescriptors().get(entry.getKey()));
            String value = (String) propertyMap.get(entry.getKey());
            if (StringUtils.isBlank(value)) {
                value = (String) propertyMap.get(controllerServiceDTO.getDescriptors().get(entry.getKey()).getDisplayName().toLowerCase());
            }
            if (!isSensitive || (isSensitive && StringUtils.isNotBlank(value))) {
                entry.setValue(value);
                changedProperties.add(entry.getKey());
            }
        });
    }
    return !changedProperties.isEmpty();
}

Example 2 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateCreationHelper method updateControllerServiceReferences.

/**
 * Fix references to the controller services on the processor properties
 *
 * @param processors                  processors to inspect
 * @param controllerServiceProperties property overrides for controller services
 * @return the list of properties that were modified
 */
public List<NifiProperty> updateControllerServiceReferences(List<ProcessorDTO> processors, Map<String, String> controllerServiceProperties, TemplateInstance instance) {
    try {
        processors = reassignControllerServiceIds(processors, instance);
        // merge the snapshotted services with the newly created ones and update respective processors in the newly created flow
        final Map<String, ControllerServiceDTO> enabledServices = new HashMap<>();
        Map<String, ControllerServiceDTO> allServices = mergedControllerServices;
        for (ControllerServiceDTO dto : allServices.values()) {
            if (NifiProcessUtil.SERVICE_STATE.ENABLED.name().equals(dto.getState())) {
                enabledServices.put(dto.getId(), dto);
                enabledServices.put(dto.getName(), dto);
            }
        }
        List<NifiProperty> properties = new ArrayList<>();
        Map<String, ProcessGroupDTO> processGroupDTOMap = new HashMap<>();
        for (ProcessorDTO dto : processors) {
            ProcessGroupDTO groupDTO = processGroupDTOMap.get(dto.getParentGroupId());
            if (groupDTO == null) {
                // we can create a tmp group dto here as all we need is the id
                groupDTO = new ProcessGroupDTO();
                groupDTO.setId(dto.getParentGroupId());
                groupDTO.setName(dto.getParentGroupId());
                processGroupDTOMap.put(dto.getParentGroupId(), groupDTO);
            }
            properties.addAll(NifiPropertyUtil.getPropertiesForProcessor(groupDTO, dto, restClient.getPropertyDescriptorTransform()));
        }
        List<NifiProperty> updatedProperties = fixControllerServiceReferences(controllerServiceProperties, enabledServices, allServices, properties);
        updatedProperties.forEach(property -> restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property));
        return updatedProperties;
    } catch (NifiClientRuntimeException e) {
        errors.add(new NifiError(NifiError.SEVERITY.FATAL, "Error trying to identify Controller Services. " + e.getMessage(), NifiProcessGroup.CONTROLLER_SERVICE_CATEGORY));
    }
    return Collections.emptyList();
}

Example 3 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateCreationHelper method identifyNewlyCreatedControllerServiceReferences.

/**
 * Compare the services in Nifi with the ones from the snapshot and return any that are not in the snapshot
 */
public Set<ControllerServiceDTO> identifyNewlyCreatedControllerServiceReferences() {
    Set<ControllerServiceDTO> newServices = new HashSet<>();
    Set<ControllerServiceDTO> controllerServiceEntity = restClient.getControllerServices();
    if (controllerServiceEntity != null) {
        if (snapshotControllerServices != null) {
            for (ControllerServiceDTO dto : controllerServiceEntity) {
                if (!snapshotControllerServices.contains(dto)) {
                    newServices.add(dto);
                }
            }
        } else {
            newServices = controllerServiceEntity;
        }
    }
    newlyCreatedControllerServices = newServices;
    mergeControllerServices(null);
    return newServices;
}

Example 4 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateCreationHelper method cleanupControllerServices.

public void cleanupControllerServices() {
    // only keep them if they are the first of their kind
    if (snapshotControllerServices != null && !snapshotControllerServices.isEmpty()) {
        final Set<String> serviceTypes = new HashSet<>();
        for (ControllerServiceDTO dto : snapshotControllerServices) {
            serviceTypes.add(dto.getType());
        }
        List<ControllerServiceDTO> servicesToDelete = Lists.newArrayList(Iterables.filter(newlyCreatedControllerServices, new Predicate<ControllerServiceDTO>() {

            @Override
            public boolean apply(ControllerServiceDTO controllerServiceDTO) {
                return serviceTypes.contains(controllerServiceDTO.getType()) && (controllerServiceDTO.getReferencingComponents() == null || controllerServiceDTO.getReferencingComponents().size() == 0);
            }
        }));
        if (servicesToDelete != null && !servicesToDelete.isEmpty()) {
            try {
                restClient.deleteControllerServices(servicesToDelete);
            } catch (Exception e) {
                log.info("error attempting to cleanup controller services while trying to delete Services: " + e.getMessage() + ".  It might be wise to login to NIFI and verify there are not extra controller services");
                getErrors().add(new NifiError(NifiError.SEVERITY.INFO, "There is an error attempting to remove the controller service :" + e.getMessage()));
            }
        }
    }
}

Example 5 with ControllerServiceDTO

use of org.apache.nifi.web.api.dto.ControllerServiceDTO in project kylo by Teradata.

the class TemplateCreationHelper method mergeControllerServices.

private void mergeControllerServices(TemplateInstance templateInstance) {
    final Map<String, ControllerServiceDTO> map = new HashMap<String, ControllerServiceDTO>();
    final Map<String, List<ControllerServiceDTO>> serviceNameMap = new HashMap<>();
    // first use the snapshotted servies as a baseline
    for (ControllerServiceDTO serviceDTO : snapshotControllerServices) {
        map.put(serviceDTO.getId(), serviceDTO);
        if (!serviceNameMap.containsKey(serviceDTO.getName())) {
            serviceNameMap.put(serviceDTO.getName(), new ArrayList<ControllerServiceDTO>());
        }
        serviceNameMap.get(serviceDTO.getName()).add(serviceDTO);
    }
    java.util.function.Predicate<ControllerServiceDTO> matchingServiceFilter = (cs) -> map.containsKey(cs.getId()) || serviceNameMap.containsKey(cs.getName());
    List<ControllerServiceDTO> matchingControllerServices = newlyCreatedControllerServices.stream().filter(matchingServiceFilter).collect(Collectors.toList());
    List<ControllerServiceDTO> unmatchedServices = newlyCreatedControllerServices.stream().filter(matchingServiceFilter.negate()).collect(Collectors.toList());
    // if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id.
    if (unmatchedServices != null && !unmatchedServices.isEmpty()) {
        Map<String, ControllerServiceDTO> updatedServices = unmatchedServices.stream().map(serviceToAdd -> {
            // if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id
            if (serviceToAdd.getDescriptors() != null && serviceToAdd.getDescriptors().values().stream().anyMatch(propertyDescriptorDTO -> StringUtils.isNotBlank(propertyDescriptorDTO.getIdentifiesControllerService()))) {
                try {
                    Optional<ControllerServiceDTO> cs = restClient.getNiFiRestClient().controllerServices().findById(serviceToAdd.getId());
                    if (cs.isPresent()) {
                        return cs.get();
                    } else {
                        return serviceToAdd;
                    }
                } catch (Exception e) {
                    return serviceToAdd;
                }
            } else {
                return serviceToAdd;
            }
        }).collect(Collectors.toMap(service -> service.getId(), service -> service));
        map.putAll(updatedServices);
        // update the core item
        newlyCreatedControllerServices = newlyCreatedControllerServices.stream().map(controllerServiceDTO -> {
            if (map.containsKey(controllerServiceDTO.getId())) {
                return updatedServices.get(controllerServiceDTO.getId());
            } else {
                return controllerServiceDTO;
            }
        }).collect(Collectors.toSet());
    }
    // if match existing services, then delete the new ones
    if (matchingControllerServices != null && !matchingControllerServices.isEmpty()) {
        for (ControllerServiceDTO serviceToDelete : matchingControllerServices) {
            try {
                if (templateInstance != null) {
                    templateInstance.addDeletedServiceMapping(serviceToDelete.getId(), serviceNameMap.get(serviceToDelete.getName()));
                }
                restClient.deleteControllerService(serviceToDelete.getId());
            } catch (NifiClientRuntimeException e) {
                log.error("Exception while attempting to mergeControllerServices.  Unable to delete Service {}. {}", serviceToDelete.getId(), e.getMessage());
            }
        }
    }
    mergedControllerServices = map;
    // validate
    // Create a map of the Controller Service Name to list of matching services
    this.serviceNameMap = mergedControllerServices.values().stream().collect(Collectors.groupingBy(cs -> cs.getName()));
    this.enabledServiceNameMap = mergedControllerServices.values().stream().filter(cs -> NifiProcessUtil.SERVICE_STATE.ENABLED.name().equalsIgnoreCase(cs.getState())).collect(Collectors.groupingBy(cs -> cs.getName()));
}