Search in sources :

Example 1 with ConnectorPluginNiFiControllerService

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService in project kylo by Teradata.

the class DataSourceProvider method createOrUpdateNiFiControllerService.

/**
 * Creates or updates the NiFi controller service linked to the specified data source.
 */
private void createOrUpdateNiFiControllerService(@Nonnull final DataSource dataSource, @Nonnull final ConnectorPluginDescriptor connectorPluginDescriptor, boolean checkExisting) throws PotentialControllerServiceConflictException {
    ConnectorPluginNiFiControllerService plugin = connectorPluginDescriptor.getNifiControllerService();
    // Resolve properties
    final PropertyPlaceholderHelper.PlaceholderResolver resolver = new DataSourcePlaceholderResolver(dataSource);
    final Map<String, String> properties = new HashMap<>(plugin.getProperties().size());
    final Map<String, ConnectorPluginNiFiControllerServicePropertyDescriptor> propertyDescriptors = plugin.getPropertyDescriptors();
    plugin.getProperties().forEach((key, value) -> {
        final String resolvedValue = placeholderHelper.replacePlaceholders(value, resolver);
        // set empty string to null
        ConnectorPluginNiFiControllerServicePropertyDescriptor descriptor = propertyDescriptors != null ? propertyDescriptors.get(key) : null;
        if (StringUtils.isBlank(resolvedValue) && (descriptor == null || (descriptor != null && !descriptor.isEmptyStringIfNull()))) {
            // set the value to null if its not explicitly configured to be set to an empty string
            properties.put(key, null);
        } else if (resolvedValue != null && !resolvedValue.startsWith("{cipher}")) {
            properties.put(key, resolvedValue);
        }
    });
    // Update or create the controller service
    ControllerServiceDTO controllerService = null;
    if (dataSource.getNifiControllerServiceId() != null && dataSource.getId() != null) {
        controllerService = new ControllerServiceDTO();
        controllerService.setId(dataSource.getNifiControllerServiceId());
        controllerService.setName(dataSource.getTitle());
        controllerService.setProperties(properties);
        try {
            controllerService = nifiRestClient.controllerServices().updateServiceAndReferencingComponents(controllerService);
            dataSource.setNifiControllerServiceId(controllerService.getId());
        } catch (final NifiComponentNotFoundException e) {
            log.warn("Controller service is missing for data source: {}", dataSource.getId(), e);
            controllerService = null;
        }
    }
    if (controllerService == null && StringUtils.isBlank(dataSource.getNifiControllerServiceId())) {
        if (checkExisting) {
            List<ControllerServiceDTO> matchingServices = findMatchingControllerService(dataSource, properties, connectorPluginDescriptor);
            if (matchingServices != null && !matchingServices.isEmpty()) {
                Map<String, String> identityProperties = plugin.getIdentityProperties().stream().collect(Collectors.toMap(propertyKey -> propertyKey, propertyKey -> properties.get(propertyKey)));
                throw new PotentialControllerServiceConflictException(new ControllerServiceConflictEntity(dataSource.getTitle(), identityProperties, matchingServices));
            }
        }
        controllerService = new ControllerServiceDTO();
        controllerService.setType(plugin.getType());
        controllerService.setName(dataSource.getTitle());
        controllerService.setProperties(properties);
        controllerService = nifiRestClient.controllerServices().create(controllerService);
        try {
            nifiRestClient.controllerServices().updateStateById(controllerService.getId(), NiFiControllerServicesRestClient.State.ENABLED);
        } catch (final NifiClientRuntimeException e) {
            log.error("Failed to enable controller service for data source: {}", dataSource.getId(), e);
            nifiRestClient.controllerServices().disableAndDeleteAsync(controllerService.getId());
            throw e;
        }
        dataSource.setNifiControllerServiceId(controllerService.getId());
    }
}
Also used : NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) DefaultDataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) DatasourceCriteria(com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Map(java.util.Map) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Pageable(org.springframework.data.domain.Pageable) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) NiFiControllerServicesRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiControllerServicesRestClient) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) PropertyPlaceholderHelper(org.springframework.util.PropertyPlaceholderHelper) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) Predicate(java.util.function.Predicate) Page(org.springframework.data.domain.Page) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) ConnectorPluginNiFiControllerService(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService) Optional(java.util.Optional) PageImpl(org.springframework.data.domain.PageImpl) ConnectorPlugin(com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin) ConnectorPluginManager(com.thinkbiganalytics.kylo.catalog.ConnectorPluginManager) HashMap(java.util.HashMap) Function(java.util.function.Function) Supplier(java.util.function.Supplier) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) ConnectorPluginNiFiControllerServicePropertyDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) CatalogException(com.thinkbiganalytics.kylo.catalog.CatalogException) DataSourceCredentialManager(com.thinkbiganalytics.kylo.catalog.credential.api.DataSourceCredentialManager) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ConnectorProvider(com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider) UserDatasource(com.thinkbiganalytics.metadata.api.datasource.UserDatasource) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) SecurityContextUtil(com.thinkbiganalytics.security.context.SecurityContextUtil) Logger(org.slf4j.Logger) Consumer(java.util.function.Consumer) Component(org.springframework.stereotype.Component) Connector(com.thinkbiganalytics.kylo.catalog.rest.model.Connector) ConnectorPluginDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor) Comparator(java.util.Comparator) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) HashMap(java.util.HashMap) ConnectorPluginNiFiControllerServicePropertyDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) ConnectorPluginNiFiControllerService(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService) PropertyPlaceholderHelper(org.springframework.util.PropertyPlaceholderHelper)

Example 2 with ConnectorPluginNiFiControllerService

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService in project kylo by Teradata.

the class DataSourceProvider method findMatchingControllerService.

private List<ControllerServiceDTO> findMatchingControllerService(@Nonnull final DataSource dataSource, @Nonnull final Map<String, String> properties, @Nonnull final ConnectorPluginDescriptor connectorPluginDescriptor) {
    ConnectorPluginNiFiControllerService plugin = connectorPluginDescriptor.getNifiControllerService();
    String type = plugin.getType();
    Map<String, String> identityProperties = plugin.getIdentityProperties().stream().collect(Collectors.toMap(propertyKey -> propertyKey, propertyKey -> properties.get(propertyKey)));
    // TODO hit cache first
    String rootProcessGroupId = nifiRestClient.processGroups().findRoot().getId();
    return nifiRestClient.processGroups().getControllerServices(rootProcessGroupId).stream().filter(controllerServiceDTO -> isMatch(controllerServiceDTO, type, dataSource.getTitle(), identityProperties)).collect(Collectors.toList());
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) DefaultDataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) DatasourceCriteria(com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Map(java.util.Map) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Pageable(org.springframework.data.domain.Pageable) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) NiFiControllerServicesRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiControllerServicesRestClient) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) PropertyPlaceholderHelper(org.springframework.util.PropertyPlaceholderHelper) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) Predicate(java.util.function.Predicate) Page(org.springframework.data.domain.Page) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) ConnectorPluginNiFiControllerService(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService) Optional(java.util.Optional) PageImpl(org.springframework.data.domain.PageImpl) ConnectorPlugin(com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin) ConnectorPluginManager(com.thinkbiganalytics.kylo.catalog.ConnectorPluginManager) HashMap(java.util.HashMap) Function(java.util.function.Function) Supplier(java.util.function.Supplier) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) ConnectorPluginNiFiControllerServicePropertyDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) CatalogException(com.thinkbiganalytics.kylo.catalog.CatalogException) DataSourceCredentialManager(com.thinkbiganalytics.kylo.catalog.credential.api.DataSourceCredentialManager) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ConnectorProvider(com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider) UserDatasource(com.thinkbiganalytics.metadata.api.datasource.UserDatasource) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) SecurityContextUtil(com.thinkbiganalytics.security.context.SecurityContextUtil) Logger(org.slf4j.Logger) Consumer(java.util.function.Consumer) Component(org.springframework.stereotype.Component) Connector(com.thinkbiganalytics.kylo.catalog.rest.model.Connector) ConnectorPluginDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor) Comparator(java.util.Comparator) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) ConnectorPluginNiFiControllerService(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService)

Aggregations

DatasourceModelTransform (com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform)2 CatalogException (com.thinkbiganalytics.kylo.catalog.CatalogException)2 ConnectorPluginManager (com.thinkbiganalytics.kylo.catalog.ConnectorPluginManager)2 DataSourceCredentialManager (com.thinkbiganalytics.kylo.catalog.credential.api.DataSourceCredentialManager)2 CatalogModelTransform (com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform)2 Connector (com.thinkbiganalytics.kylo.catalog.rest.model.Connector)2 ConnectorPluginDescriptor (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor)2 ConnectorPluginNiFiControllerService (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService)2 ConnectorPluginNiFiControllerServicePropertyDescriptor (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor)2 DataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate)2 DataSource (com.thinkbiganalytics.kylo.catalog.rest.model.DataSource)2 DefaultDataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate)2 ConnectorPlugin (com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin)2 MetadataAccess (com.thinkbiganalytics.metadata.api.MetadataAccess)2 ConnectorProvider (com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider)2 Datasource (com.thinkbiganalytics.metadata.api.datasource.Datasource)2 DatasourceCriteria (com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria)2 DatasourceProvider (com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider)2 UserDatasource (com.thinkbiganalytics.metadata.api.datasource.UserDatasource)2 JdbcDatasource (com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource)2