Examples with ProcessorConfigDTO org.apache.nifi.web.api.dto.ProcessorConfigDTO used on opensource projects

Example 16 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project kylo by Teradata.

the class TemplateCreationHelperTest method updateControllerServiceReferencesWithEnabled.

/**
 * Verify preferring enabled controller services over disabled controller services when updating processor properties.
 */
@Test
public void updateControllerServiceReferencesWithEnabled() {
    final AtomicReference<NifiProperty> updateProperty = new AtomicReference<>();
    // Mock NiFi client
    final LegacyNifiRestClient restClient = Mockito.mock(LegacyNifiRestClient.class);
    Mockito.when(restClient.getPropertyDescriptorTransform()).thenReturn(new MockNiFiPropertyDescriptorTransform());
    Mockito.doAnswer(invocation -> {
        updateProperty.set(invocation.getArgumentAt(2, NifiProperty.class));
        return null;
    }).when(restClient).updateProcessorProperty(Mockito.isNull(String.class), Mockito.eq("P1"), Mockito.any());
    final ControllerServiceDTO service1 = new ControllerServiceDTO();
    service1.setId("S1");
    service1.setName("Service1");
    service1.setProperties(Collections.emptyMap());
    service1.setState("DISABLED");
    final ControllerServiceDTO service2 = new ControllerServiceDTO();
    service2.setId("S2");
    service2.setName("Service2");
    service2.setProperties(Collections.emptyMap());
    service2.setState("ENABLED");
    Mockito.when(restClient.getControllerServices()).thenReturn(ImmutableSet.of(service1, service2));
    // Mock processors
    final ProcessorConfigDTO config = new ProcessorConfigDTO();
    config.setDescriptors(Collections.singletonMap("service", newPropertyDescriptor("service", "com.example.Service", "S1", "S2")));
    config.setProperties(Collections.singletonMap("service", "invalid"));
    final ProcessorDTO processor = new ProcessorDTO();
    processor.setId("P1");
    processor.setName("Processor1");
    processor.setConfig(config);
    // Update processors
    final TemplateCreationHelper helper = new TemplateCreationHelper(restClient);
    helper.snapshotControllerServiceReferences();
    helper.identifyNewlyCreatedControllerServiceReferences();
    helper.updateControllerServiceReferences(Collections.singletonList(processor));
    // Verify new processor properties
    Assert.assertNotNull("Property 'Service' not set on processor 'Processor1'.", updateProperty.get());
    Assert.assertEquals("S2", updateProperty.get().getValue());
}

Example 17 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project kylo by Teradata.

the class TemplateCreationHelper method reassignControllerServiceIds.

private List<ProcessorDTO> reassignControllerServiceIds(List<ProcessorDTO> processors, TemplateInstance instance) {
    Set<ProcessorDTO> updatedProcessors = new HashSet<>();
    if (processors != null) {
        processors.stream().forEach(processorDTO -> {
            Map<String, String> updatedProcessorProperties = new HashMap<>();
            processorDTO.getConfig().getDescriptors().forEach((k, v) -> {
                if (v.getIdentifiesControllerService() != null) {
                    boolean idsMatch = getMergedControllerServices().keySet().stream().anyMatch(id -> id.equalsIgnoreCase(processorDTO.getConfig().getProperties().get(k)));
                    if (!idsMatch && templateProperties != null && !templateProperties.isEmpty()) {
                        NifiProperty matchingProperty = templateProperties.stream().filter(p -> p.getKey().equalsIgnoreCase(k) && p.getProcessorName().equalsIgnoreCase(processorDTO.getName()) && v.getIdentifiesControllerService().equalsIgnoreCase(p.getPropertyDescriptor().getIdentifiesControllerService())).findFirst().orElse(null);
                        if (matchingProperty != null && matchingProperty.getPropertyDescriptor() != null && matchingProperty.getPropertyDescriptor().getAllowableValues() != null) {
                            NiFiAllowableValue matchingValue = matchingProperty.getPropertyDescriptor().getAllowableValues().stream().filter(niFiAllowableValue -> niFiAllowableValue.getValue().equalsIgnoreCase(matchingProperty.getValue())).findFirst().orElse(null);
                            if (matchingValue != null) {
                                String name = matchingValue.getDisplayName();
                                String validControllerServiceId = hasMatchingService(enabledServiceNameMap, name) ? enabledServiceNameMap.get(name).get(0).getId() : hasMatchingService(serviceNameMap, name) ? serviceNameMap.get(name).get(0).getId() : null;
                                if (StringUtils.isNotBlank(validControllerServiceId) && (v.isRequired() || !v.isRequired() && StringUtils.isNotBlank(processorDTO.getConfig().getProperties().get(k)))) {
                                    processorDTO.getConfig().getProperties().put(k, validControllerServiceId);
                                    updatedProcessorProperties.put(k, validControllerServiceId);
                                    if (!updatedProcessors.contains(processorDTO)) {
                                        updatedProcessors.add(processorDTO);
                                    }
                                }
                            }
                        }
                    }
                    // if we havent made a match attempt to see if the cs was removed
                    if (!updatedProcessorProperties.containsKey(k) && !idsMatch && instance != null) {
                        String value = processorDTO.getConfig().getProperties().get(k);
                        // find the correct reference from that was removed due to a matching service
                        ControllerServiceDTO controllerServiceDTO = instance.findMatchingControllerServoce(value);
                        if (controllerServiceDTO != null) {
                            updatedProcessorProperties.put(k, controllerServiceDTO.getId());
                        }
                    }
                }
            });
            if (!updatedProcessorProperties.isEmpty()) {
                ProcessorDTO updatedProcessor = new ProcessorDTO();
                updatedProcessor.setId(processorDTO.getId());
                updatedProcessor.setConfig(new ProcessorConfigDTO());
                updatedProcessor.getConfig().setProperties(updatedProcessorProperties);
                // update the processor
                ProcessorDTO updated = restClient.updateProcessor(updatedProcessor);
                updatedProcessors.add(updated);
            }
        });
    }
    // update the data back in the processors list
    if (!updatedProcessors.isEmpty()) {
        Map<String, ProcessorDTO> updatedMap = updatedProcessors.stream().collect(Collectors.toMap(p -> p.getId(), p -> p));
        return processors.stream().map(p -> updatedMap.containsKey(p.getId()) ? updatedMap.get(p.getId()) : p).collect(Collectors.toList());
    }
    return processors;
}

Example 18 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project kylo by Teradata.

the class AbstractNiFiProcessorsRestClient method applySchedule.

protected ProcessorDTO applySchedule(NifiProcessorSchedule schedule) {
    if (schedule != null && schedule.getProcessorId() != null) {
        ProcessorDTO input = new ProcessorDTO();
        input.setId(schedule.getProcessorId());
        input.setConfig(new ProcessorConfigDTO());
        input.getConfig().setSchedulingPeriod(schedule.getSchedulingPeriod());
        input.getConfig().setSchedulingStrategy(schedule.getSchedulingStrategy());
        input.getConfig().setConcurrentlySchedulableTaskCount(schedule.getConcurrentTasks());
        return input;
    }
    return null;
}

Example 19 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class StandardNiFiServiceFacade method ensureDefaultPropertyValuesArePopulated.

/**
 * Ensures default values are populated for all components in this snippet. This is necessary to handle old templates without default values
 * and when existing properties have default values introduced.
 *
 * @param snippet snippet
 */
private void ensureDefaultPropertyValuesArePopulated(final FlowSnippetDTO snippet) {
    if (snippet != null) {
        if (snippet.getControllerServices() != null) {
            snippet.getControllerServices().forEach(dto -> {
                if (dto.getProperties() == null) {
                    dto.setProperties(new LinkedHashMap<>());
                }
                try {
                    final ConfigurableComponent configurableComponent = controllerFacade.getTemporaryComponent(dto.getType(), dto.getBundle());
                    configurableComponent.getPropertyDescriptors().forEach(descriptor -> {
                        if (dto.getProperties().get(descriptor.getName()) == null) {
                            dto.getProperties().put(descriptor.getName(), descriptor.getDefaultValue());
                        }
                    });
                } catch (final Exception e) {
                    logger.warn(String.format("Unable to create ControllerService of type %s to populate default values.", dto.getType()));
                }
            });
        }
        if (snippet.getProcessors() != null) {
            snippet.getProcessors().forEach(dto -> {
                if (dto.getConfig() == null) {
                    dto.setConfig(new ProcessorConfigDTO());
                }
                final ProcessorConfigDTO config = dto.getConfig();
                if (config.getProperties() == null) {
                    config.setProperties(new LinkedHashMap<>());
                }
                try {
                    final ConfigurableComponent configurableComponent = controllerFacade.getTemporaryComponent(dto.getType(), dto.getBundle());
                    configurableComponent.getPropertyDescriptors().forEach(descriptor -> {
                        if (config.getProperties().get(descriptor.getName()) == null) {
                            config.getProperties().put(descriptor.getName(), descriptor.getDefaultValue());
                        }
                    });
                } catch (final Exception e) {
                    logger.warn(String.format("Unable to create Processor of type %s to populate default values.", dto.getType()));
                }
            });
        }
        if (snippet.getProcessGroups() != null) {
            snippet.getProcessGroups().forEach(processGroup -> {
                ensureDefaultPropertyValuesArePopulated(processGroup.getContents());
            });
        }
    }
}

Example 20 with ProcessorConfigDTO

use of org.apache.nifi.web.api.dto.ProcessorConfigDTO in project nifi by apache.

the class TestFlowController method testInstantiateSnippetWithProcessor.

@Test
public void testInstantiateSnippetWithProcessor() throws ProcessorInstantiationException {
    final String id = UUID.randomUUID().toString();
    final BundleCoordinate coordinate = systemBundle.getBundleDetails().getCoordinate();
    final ProcessorNode processorNode = controller.createProcessor(DummyProcessor.class.getName(), id, coordinate);
    // create a processor dto
    final ProcessorDTO processorDTO = new ProcessorDTO();
    // use a different id here
    processorDTO.setId(UUID.randomUUID().toString());
    processorDTO.setPosition(new PositionDTO(new Double(0), new Double(0)));
    processorDTO.setStyle(processorNode.getStyle());
    processorDTO.setParentGroupId("1234");
    processorDTO.setInputRequirement(processorNode.getInputRequirement().name());
    processorDTO.setPersistsState(processorNode.getProcessor().getClass().isAnnotationPresent(Stateful.class));
    processorDTO.setRestricted(processorNode.isRestricted());
    processorDTO.setExtensionMissing(processorNode.isExtensionMissing());
    processorDTO.setType(processorNode.getCanonicalClassName());
    processorDTO.setBundle(new BundleDTO(coordinate.getGroup(), coordinate.getId(), coordinate.getVersion()));
    processorDTO.setName(processorNode.getName());
    processorDTO.setState(processorNode.getScheduledState().toString());
    processorDTO.setRelationships(new ArrayList<>());
    processorDTO.setDescription("description");
    processorDTO.setSupportsParallelProcessing(!processorNode.isTriggeredSerially());
    processorDTO.setSupportsEventDriven(processorNode.isEventDrivenSupported());
    processorDTO.setSupportsBatching(processorNode.isSessionBatchingSupported());
    ProcessorConfigDTO configDTO = new ProcessorConfigDTO();
    configDTO.setSchedulingPeriod(processorNode.getSchedulingPeriod());
    configDTO.setPenaltyDuration(processorNode.getPenalizationPeriod());
    configDTO.setYieldDuration(processorNode.getYieldPeriod());
    configDTO.setRunDurationMillis(processorNode.getRunDuration(TimeUnit.MILLISECONDS));
    configDTO.setConcurrentlySchedulableTaskCount(processorNode.getMaxConcurrentTasks());
    configDTO.setLossTolerant(processorNode.isLossTolerant());
    configDTO.setComments(processorNode.getComments());
    configDTO.setBulletinLevel(processorNode.getBulletinLevel().name());
    configDTO.setSchedulingStrategy(processorNode.getSchedulingStrategy().name());
    configDTO.setExecutionNode(processorNode.getExecutionNode().name());
    configDTO.setAnnotationData(processorNode.getAnnotationData());
    processorDTO.setConfig(configDTO);
    // create the snippet with the processor
    final FlowSnippetDTO flowSnippetDTO = new FlowSnippetDTO();
    flowSnippetDTO.setProcessors(Collections.singleton(processorDTO));
    // instantiate the snippet
    assertEquals(0, controller.getRootGroup().getProcessors().size());
    controller.instantiateSnippet(controller.getRootGroup(), flowSnippetDTO);
    assertEquals(1, controller.getRootGroup().getProcessors().size());
}