Examples with NifiProperty com.thinkbiganalytics.nifi.rest.model.NifiProperty used on opensource projects

Example 1 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelper method updateControllerServiceReferences.

/**
 * Fix references to the controller services on the processor properties
 *
 * @param processors                  processors to inspect
 * @param controllerServiceProperties property overrides for controller services
 * @return the list of properties that were modified
 */
public List<NifiProperty> updateControllerServiceReferences(List<ProcessorDTO> processors, Map<String, String> controllerServiceProperties, TemplateInstance instance) {
    try {
        processors = reassignControllerServiceIds(processors, instance);
        // merge the snapshotted services with the newly created ones and update respective processors in the newly created flow
        final Map<String, ControllerServiceDTO> enabledServices = new HashMap<>();
        Map<String, ControllerServiceDTO> allServices = mergedControllerServices;
        for (ControllerServiceDTO dto : allServices.values()) {
            if (NifiProcessUtil.SERVICE_STATE.ENABLED.name().equals(dto.getState())) {
                enabledServices.put(dto.getId(), dto);
                enabledServices.put(dto.getName(), dto);
            }
        }
        List<NifiProperty> properties = new ArrayList<>();
        Map<String, ProcessGroupDTO> processGroupDTOMap = new HashMap<>();
        for (ProcessorDTO dto : processors) {
            ProcessGroupDTO groupDTO = processGroupDTOMap.get(dto.getParentGroupId());
            if (groupDTO == null) {
                // we can create a tmp group dto here as all we need is the id
                groupDTO = new ProcessGroupDTO();
                groupDTO.setId(dto.getParentGroupId());
                groupDTO.setName(dto.getParentGroupId());
                processGroupDTOMap.put(dto.getParentGroupId(), groupDTO);
            }
            properties.addAll(NifiPropertyUtil.getPropertiesForProcessor(groupDTO, dto, restClient.getPropertyDescriptorTransform()));
        }
        List<NifiProperty> updatedProperties = fixControllerServiceReferences(controllerServiceProperties, enabledServices, allServices, properties);
        updatedProperties.forEach(property -> restClient.updateProcessorProperty(property.getProcessGroupId(), property.getProcessorId(), property));
        return updatedProperties;
    } catch (NifiClientRuntimeException e) {
        errors.add(new NifiError(NifiError.SEVERITY.FATAL, "Error trying to identify Controller Services. " + e.getMessage(), NifiProcessGroup.CONTROLLER_SERVICE_CATEGORY));
    }
    return Collections.emptyList();
}

Example 2 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelper method tryToEnableControllerService.

/**
 * Tries to enable the specified controller service.
 *
 * @param controllerService the controller service to enable
 * @param properties        property overrides for the controller service
 * @param enabledServices   map of enabled controller service ids and names to DTOs
 * @param allServices       map of all controller service ids to
 * @return the enabled controller service
 * @throws NifiComponentNotFoundException if the controller service does not exist
 * @throws WebApplicationException        if the controller service cannot be enabled
 */
@Nonnull
private ControllerServiceDTO tryToEnableControllerService(@Nonnull final ControllerServiceDTO controllerService, @Nullable final Map<String, String> properties, @Nonnull final Map<String, ControllerServiceDTO> enabledServices, @Nonnull final Map<String, ControllerServiceDTO> allServices) {
    // Check if already enabled
    if ("ENABLED".equals(controllerService.getState())) {
        return controllerService;
    }
    // Fix controller service references
    final NiFiPropertyDescriptorTransform propertyDescriptorTransform = restClient.getPropertyDescriptorTransform();
    final List<NifiProperty> changedProperties = fixControllerServiceReferences(properties, enabledServices, allServices, NifiPropertyUtil.getPropertiesForService(controllerService, propertyDescriptorTransform));
    if (!changedProperties.isEmpty()) {
        changedProperties.forEach(property -> {
            controllerService.getProperties().put(property.getKey(), property.getValue());
        });
        nifiRestClient.controllerServices().update(controllerService);
    }
    // Enable controller service
    return restClient.enableControllerServiceAndSetProperties(controllerService.getId(), properties);
}

Example 3 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelper method fixControllerServiceReferences.

/**
 * Enables the controller services for the specified properties or changes the property value to an enabled service.
 *
 * @param controllerServiceProperties property overrides for controller services
 * @param enabledServices             map of enabled controller service ids and names to DTOs
 * @param allServices                 map of all controller service ids to DTOs
 * @param properties                  the processor properties to update
 * @return the list of properties that were modified
 */
@Nonnull
private List<NifiProperty> fixControllerServiceReferences(@Nullable final Map<String, String> controllerServiceProperties, @Nonnull final Map<String, ControllerServiceDTO> enabledServices, @Nonnull final Map<String, ControllerServiceDTO> allServices, @Nonnull final List<NifiProperty> properties) {
    return properties.stream().filter(property -> StringUtils.isNotBlank(property.getPropertyDescriptor().getIdentifiesControllerService())).filter(property -> StringUtils.isNotBlank(property.getValue())).filter(property -> !enabledServices.containsKey(property.getValue())).filter(property -> {
        final Optional<ControllerServiceDTO> controllerService = findControllerServiceForProperty(controllerServiceProperties, enabledServices, allServices, property);
        if (controllerService.isPresent()) {
            if (!controllerService.get().getId().equals(property.getValue())) {
                property.setValue(controllerService.get().getId());
                return true;
            }
        } else if (property.getPropertyDescriptor().isRequired()) {
            final String message = "Unable to find a valid controller service for the '" + property.getKey() + "' property of the '" + property.getProcessorName() + "' " + "processor.";
            errors.add(new NifiError(NifiError.SEVERITY.FATAL, message, NifiProcessGroup.CONTROLLER_SERVICE_CATEGORY));
        }
        return false;
    }).collect(Collectors.toList());
}

Example 4 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelperTest method updateControllerServiceReferencesWithRecursive.

/**
 * Verify recursively enabling controller services when updating processor properties.
 */
@Test
public void updateControllerServiceReferencesWithRecursive() {
    final List<ControllerServiceDTO> updatedControllerServices = new ArrayList<>();
    final List<NifiProperty> updatedProperties = new ArrayList<>();
    // Mock NiFi client
    final NiFiControllerServicesRestClient controllerServicesRestClient = Mockito.mock(NiFiControllerServicesRestClient.class);
    Mockito.when(controllerServicesRestClient.update(Mockito.any())).thenAnswer(answer -> {
        final ControllerServiceDTO controllerService = answer.getArgumentAt(0, ControllerServiceDTO.class);
        updatedControllerServices.add(controllerService);
        return controllerService;
    });
    final NiFiRestClient restClient = Mockito.mock(NiFiRestClient.class);
    Mockito.when(restClient.controllerServices()).thenReturn(controllerServicesRestClient);
    // Mock Legacy NiFi client
    final LegacyNifiRestClient legacyRestClient = Mockito.mock(LegacyNifiRestClient.class);
    Mockito.when(legacyRestClient.enableControllerServiceAndSetProperties(Mockito.any(), Mockito.any())).thenReturn(new ControllerServiceDTO());
    Mockito.when(legacyRestClient.getNiFiRestClient()).thenReturn(restClient);
    Mockito.when(legacyRestClient.getPropertyDescriptorTransform()).thenReturn(new MockNiFiPropertyDescriptorTransform());
    Mockito.doAnswer(invocation -> {
        updatedProperties.add(invocation.getArgumentAt(2, NifiProperty.class));
        return null;
    }).when(legacyRestClient).updateProcessorProperty(Mockito.isNull(String.class), Mockito.eq("P1"), Mockito.any());
    final ControllerServiceDTO service1 = new ControllerServiceDTO();
    service1.setDescriptors(Collections.singletonMap("service", newPropertyDescriptor("service", "com.example.Service2", "S2")));
    service1.setId("S1");
    service1.setName("Service1");
    service1.setProperties(newHashMap("service", "invalid"));
    service1.setState("DISABLED");
    final ControllerServiceDTO service2 = new ControllerServiceDTO();
    service2.setId("S2");
    service2.setName("Service2");
    service2.setProperties(new HashMap<>());
    service2.setState("ENABLED");
    Mockito.when(legacyRestClient.getControllerServices()).thenReturn(ImmutableSet.of(service1, service2));
    // Mock processors
    final ProcessorConfigDTO config = new ProcessorConfigDTO();
    config.setDescriptors(Collections.singletonMap("service", newPropertyDescriptor("service", "com.example.Service1", "S1")));
    config.setProperties(Collections.singletonMap("service", "invalid"));
    final ProcessorDTO processor = new ProcessorDTO();
    processor.setId("P1");
    processor.setName("Processor1");
    processor.setConfig(config);
    // Update processors
    final TemplateCreationHelper helper = new TemplateCreationHelper(legacyRestClient);
    helper.snapshotControllerServiceReferences();
    helper.identifyNewlyCreatedControllerServiceReferences();
    helper.updateControllerServiceReferences(Collections.singletonList(processor));
    // Verify updated properties
    Assert.assertEquals("Property 'Service' not set on controller service 'Server1'.", 1, updatedControllerServices.size());
    Assert.assertEquals("S2", updatedControllerServices.get(0).getProperties().get("service"));
    Assert.assertEquals("Property 'Service' not set on processor 'Processor1'.", 1, updatedProperties.size());
    Assert.assertEquals("S1", updatedProperties.get(0).getValue());
}

Example 5 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class NifiPropertyUtil method getPropertiesForTemplate.

/**
 * For a given template object return the list of properties. optionally choose to exclude any input processors from the property list
 *
 * @param parentProcessGroup          the parent process group associated wiht the template
 * @param dto                         the template
 * @param propertyDescriptorTransform transformation utility
 * @param excludeInputProcessors      {@code true} removes the properties part of the input processors, {@code false} will include all properties in all processors of the template
 */
public static List<NifiProperty> getPropertiesForTemplate(ProcessGroupDTO parentProcessGroup, TemplateDTO dto, NiFiPropertyDescriptorTransform propertyDescriptorTransform, boolean excludeInputProcessors) {
    List<NifiProperty> properties = new ArrayList<NifiProperty>();
    if (dto != null) {
        List<ProcessorDTO> inputs = NifiTemplateUtil.getInputProcessorsForTemplate(dto);
        Set<ProcessorDTO> processorDTOSet = NifiProcessUtil.getProcessors(dto, excludeInputProcessors);
        Map<String, ProcessGroupDTO> groupMap = NifiProcessUtil.getProcessGroupsMap(dto);
        for (ProcessorDTO processor : processorDTOSet) {
            ProcessGroupDTO group = groupMap.get(processor.getParentGroupId());
            if (group == null) {
                group = parentProcessGroup;
            }
            List<NifiProperty> propertyList = getPropertiesForProcessor(group, processor, propertyDescriptorTransform);
            // assign the property as an input property if it is one
            if (NifiProcessUtil.findFirstProcessorsById(inputs, processor.getId()) != null) {
                for (NifiProperty property : propertyList) {
                    property.setInputProperty(true);
                }
            }
            properties.addAll(propertyList);
        }
    }
    return properties;
}