Examples with NifiProperty com.thinkbiganalytics.nifi.rest.model.NifiProperty used on opensource projects

Example 21 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class PropertyExpressionResolverTest method resolveExpression.

/**
 * Verifies resolving expressions in property values.
 */
@Test
public void resolveExpression() {
    final FeedMetadata metadata = new FeedMetadata();
    metadata.setSystemFeedName("myfeed");
    // Verify config variable
    final NifiProperty prop1 = createProperty("${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop1));
    Assert.assertEquals("hello-world", prop1.getValue());
    // Verify metadata variable
    final NifiProperty prop2 = createProperty("${metadata.systemFeedName}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop2));
    Assert.assertEquals("myfeed", prop2.getValue());
    // Verify static config
    final NifiProperty prop3 = createProperty(STATIC_KEY, "${metadata.systemFeedName}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop3));
    Assert.assertEquals("myapp", prop3.getValue());
    final NifiProperty prop4 = createProperty(STATIC_KEY, "${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop4));
    Assert.assertEquals("hello-world", prop4.getValue());
    final NifiProperty prop5 = createProperty(STATIC_KEY, "");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop5));
    Assert.assertEquals("myapp", prop5.getValue());
    // Verify multiple variables
    final NifiProperty prop6 = createProperty("${metadata.systemFeedName}.${config.test.value}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop6));
    Assert.assertEquals("myfeed.hello-world", prop6.getValue());
    // Verify multiple variables
    final NifiProperty prop7 = createProperty("$${${metadata.systemFeedName}.${config.test.value}}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop7));
    Assert.assertEquals("${myfeed.hello-world}", prop7.getValue());
    // Verify multiple variables
    final NifiProperty prop8 = createProperty("${config.${metadata.systemFeedName}.${config.test.value}}");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop8));
    Assert.assertEquals("runtime value", prop8.getValue());
    // Verify static text
    final NifiProperty prop9 = createProperty("config.test.value");
    Assert.assertFalse(resolver.resolveExpression(metadata, prop9));
    Assert.assertEquals("config.test.value", prop9.getValue());
    // verify replacement with NiFi el
    final NifiProperty prop10 = createProperty("property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop10));
    Assert.assertEquals("/path/to/property1,${nifi.expression.property}", prop10.getValue());
    // verify replacement without NiFi el
    final NifiProperty prop11 = createProperty("Another Processor", "property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop11));
    Assert.assertEquals("/path/to/another_processor/property1/location", prop11.getValue());
    // verify replacement without NiFi el using default processor type replacement
    final NifiProperty prop12 = createProperty("My New Processor", "property1", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, prop12));
    Assert.assertEquals("/path/to/property1/location", prop12.getValue());
    // verify replacement without NiFi el using default processor type replacement
    final NifiProperty extraFiles = createProperty("extra_files", "a value");
    Assert.assertTrue(resolver.resolveExpression(metadata, extraFiles));
    Assert.assertEquals("${table_field_policy_json_file},/usr/hdp/current/spark-client/conf/hive-site.xml", extraFiles.getValue());
    Assert.assertTrue(resolver.resolveExpression(metadata, extraFiles));
    Assert.assertEquals("${table_field_policy_json_file},/usr/hdp/current/spark-client/conf/hive-site.xml", extraFiles.getValue());
    final NifiProperty hiveSchema = createProperty(STATIC_KEY, "${config.hive.schema}");
    Assert.assertTrue(resolver.resolveExpression(metadata, hiveSchema));
    Assert.assertEquals("hive", hiveSchema.getValue());
}

Example 22 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelperTest method updateControllerServiceReferencesWithEnabled.

/**
 * Verify preferring enabled controller services over disabled controller services when updating processor properties.
 */
@Test
public void updateControllerServiceReferencesWithEnabled() {
    final AtomicReference<NifiProperty> updateProperty = new AtomicReference<>();
    // Mock NiFi client
    final LegacyNifiRestClient restClient = Mockito.mock(LegacyNifiRestClient.class);
    Mockito.when(restClient.getPropertyDescriptorTransform()).thenReturn(new MockNiFiPropertyDescriptorTransform());
    Mockito.doAnswer(invocation -> {
        updateProperty.set(invocation.getArgumentAt(2, NifiProperty.class));
        return null;
    }).when(restClient).updateProcessorProperty(Mockito.isNull(String.class), Mockito.eq("P1"), Mockito.any());
    final ControllerServiceDTO service1 = new ControllerServiceDTO();
    service1.setId("S1");
    service1.setName("Service1");
    service1.setProperties(Collections.emptyMap());
    service1.setState("DISABLED");
    final ControllerServiceDTO service2 = new ControllerServiceDTO();
    service2.setId("S2");
    service2.setName("Service2");
    service2.setProperties(Collections.emptyMap());
    service2.setState("ENABLED");
    Mockito.when(restClient.getControllerServices()).thenReturn(ImmutableSet.of(service1, service2));
    // Mock processors
    final ProcessorConfigDTO config = new ProcessorConfigDTO();
    config.setDescriptors(Collections.singletonMap("service", newPropertyDescriptor("service", "com.example.Service", "S1", "S2")));
    config.setProperties(Collections.singletonMap("service", "invalid"));
    final ProcessorDTO processor = new ProcessorDTO();
    processor.setId("P1");
    processor.setName("Processor1");
    processor.setConfig(config);
    // Update processors
    final TemplateCreationHelper helper = new TemplateCreationHelper(restClient);
    helper.snapshotControllerServiceReferences();
    helper.identifyNewlyCreatedControllerServiceReferences();
    helper.updateControllerServiceReferences(Collections.singletonList(processor));
    // Verify new processor properties
    Assert.assertNotNull("Property 'Service' not set on processor 'Processor1'.", updateProperty.get());
    Assert.assertEquals("S2", updateProperty.get().getValue());
}

Example 23 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class NifiPropertyUtil method groupPropertiesByProcessGroupAndProcessor.

/**
 * Return a map of processGroupId to a map of that groups processors and its respective propeties
 *
 * @param properties the properties to inspect
 * @return a map with the key being the processGroupId and the value being a map of properties with its key being the processorId
 */
public static Map<String, Map<String, List<NifiProperty>>> groupPropertiesByProcessGroupAndProcessor(List<NifiProperty> properties) {
    Map<String, Map<String, List<NifiProperty>>> processGroupProperties = new HashMap();
    for (NifiProperty property : properties) {
        String processGroup = property.getProcessGroupId();
        String processorId = property.getProcessorId();
        if (!processGroupProperties.containsKey(processGroup)) {
            processGroupProperties.put(processGroup, new HashMap<String, List<NifiProperty>>());
        }
        if (!processGroupProperties.get(processGroup).containsKey(processorId)) {
            processGroupProperties.get(processGroup).put(processorId, new ArrayList<NifiProperty>());
        }
        processGroupProperties.get(processGroup).get(processorId).add(property);
    }
    return processGroupProperties;
}

Example 24 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class NifiPropertyUtil method matchPropertyByProcessorName.

/**
 * update the templateProperties with those that match the nifiProperty using the {@link NifiProperty#getProcessorName()}  to make the match
 *
 * @param templateProperties the properties to update
 * @param nifiProperty       the property to check
 * @param updateMode         a mode to update
 * @return the property, or updated property if matched
 */
private static NifiProperty matchPropertyByProcessorName(Collection<NifiProperty> templateProperties, final NifiProperty nifiProperty, PROPERTY_MATCH_AND_UPDATE_MODE updateMode) {
    NifiProperty matchingProperty = findPropertyByProcessorName(templateProperties, nifiProperty);
    if (matchingProperty != null) {
        // 
        if (PROPERTY_MATCH_AND_UPDATE_MODE.FEED_DETAILS_MATCH_TEMPLATE.equals(updateMode)) {
            // copy the property
            NifiProperty copy = new NifiProperty(matchingProperty);
            templateProperties.remove(matchingProperty);
            templateProperties.add(copy);
            matchingProperty = copy;
        }
        updateMatchingProperty(matchingProperty, nifiProperty, updateMode);
    }
    return matchingProperty;
}

Example 25 with NifiProperty

use of com.thinkbiganalytics.nifi.rest.model.NifiProperty in project kylo by Teradata.

the class TemplateCreationHelper method reassignControllerServiceIds.

private List<ProcessorDTO> reassignControllerServiceIds(List<ProcessorDTO> processors, TemplateInstance instance) {
    Set<ProcessorDTO> updatedProcessors = new HashSet<>();
    if (processors != null) {
        processors.stream().forEach(processorDTO -> {
            Map<String, String> updatedProcessorProperties = new HashMap<>();
            processorDTO.getConfig().getDescriptors().forEach((k, v) -> {
                if (v.getIdentifiesControllerService() != null) {
                    boolean idsMatch = getMergedControllerServices().keySet().stream().anyMatch(id -> id.equalsIgnoreCase(processorDTO.getConfig().getProperties().get(k)));
                    if (!idsMatch && templateProperties != null && !templateProperties.isEmpty()) {
                        NifiProperty matchingProperty = templateProperties.stream().filter(p -> p.getKey().equalsIgnoreCase(k) && p.getProcessorName().equalsIgnoreCase(processorDTO.getName()) && v.getIdentifiesControllerService().equalsIgnoreCase(p.getPropertyDescriptor().getIdentifiesControllerService())).findFirst().orElse(null);
                        if (matchingProperty != null && matchingProperty.getPropertyDescriptor() != null && matchingProperty.getPropertyDescriptor().getAllowableValues() != null) {
                            NiFiAllowableValue matchingValue = matchingProperty.getPropertyDescriptor().getAllowableValues().stream().filter(niFiAllowableValue -> niFiAllowableValue.getValue().equalsIgnoreCase(matchingProperty.getValue())).findFirst().orElse(null);
                            if (matchingValue != null) {
                                String name = matchingValue.getDisplayName();
                                String validControllerServiceId = hasMatchingService(enabledServiceNameMap, name) ? enabledServiceNameMap.get(name).get(0).getId() : hasMatchingService(serviceNameMap, name) ? serviceNameMap.get(name).get(0).getId() : null;
                                if (StringUtils.isNotBlank(validControllerServiceId) && (v.isRequired() || !v.isRequired() && StringUtils.isNotBlank(processorDTO.getConfig().getProperties().get(k)))) {
                                    processorDTO.getConfig().getProperties().put(k, validControllerServiceId);
                                    updatedProcessorProperties.put(k, validControllerServiceId);
                                    if (!updatedProcessors.contains(processorDTO)) {
                                        updatedProcessors.add(processorDTO);
                                    }
                                }
                            }
                        }
                    }
                    // if we havent made a match attempt to see if the cs was removed
                    if (!updatedProcessorProperties.containsKey(k) && !idsMatch && instance != null) {
                        String value = processorDTO.getConfig().getProperties().get(k);
                        // find the correct reference from that was removed due to a matching service
                        ControllerServiceDTO controllerServiceDTO = instance.findMatchingControllerServoce(value);
                        if (controllerServiceDTO != null) {
                            updatedProcessorProperties.put(k, controllerServiceDTO.getId());
                        }
                    }
                }
            });
            if (!updatedProcessorProperties.isEmpty()) {
                ProcessorDTO updatedProcessor = new ProcessorDTO();
                updatedProcessor.setId(processorDTO.getId());
                updatedProcessor.setConfig(new ProcessorConfigDTO());
                updatedProcessor.getConfig().setProperties(updatedProcessorProperties);
                // update the processor
                ProcessorDTO updated = restClient.updateProcessor(updatedProcessor);
                updatedProcessors.add(updated);
            }
        });
    }
    // update the data back in the processors list
    if (!updatedProcessors.isEmpty()) {
        Map<String, ProcessorDTO> updatedMap = updatedProcessors.stream().collect(Collectors.toMap(p -> p.getId(), p -> p));
        return processors.stream().map(p -> updatedMap.containsKey(p.getId()) ? updatedMap.get(p.getId()) : p).collect(Collectors.toList());
    }
    return processors;
}