Examples with ConnectorPluginDescriptor com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor used on opensource projects

Example 1 with ConnectorPluginDescriptor

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor in project kylo by Teradata.

the class CatalogMetadataConfig method doPluginSyncAction.

private void doPluginSyncAction(ConnectorPluginManager pluginMgr, MetadataAccess metadata) {
    metadata.commit(() -> {
        List<ConnectorPlugin> plugins = pluginMgr.getPlugins();
        Map<String, Connector> connectorMap = connectorProvider().findAll(true).stream().collect(Collectors.toMap(Connector::getPluginId, c -> c));
        for (ConnectorPlugin plugin : plugins) {
            Connector connector = connectorMap.get(plugin.getId());
            ConnectorPluginDescriptor descr = plugin.getDescriptor();
            if (connector != null) {
                connectorMap.get(plugin.getId()).setActive(true);
                connectorMap.remove(plugin.getId());
            } else {
                String title = descr.getTitle();
                connector = connectorProvider().create(plugin.getId(), title);
            }
            connector.setIconColor(descr.getColor());
            connector.setIcon(descr.getIcon());
            if (StringUtils.isNotBlank(descr.getFormat())) {
                connector.getSparkParameters().setFormat(descr.getFormat());
            }
        }
        // least one data source, or removed if they don't.
        for (Connector connector : connectorMap.values()) {
            if (connector.getDataSources().isEmpty()) {
                connectorProvider().deleteById(connector.getId());
            } else {
                connector.setActive(false);
            }
        }
    }, MetadataAccess.SERVICE);
}

Example 2 with ConnectorPluginDescriptor

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor in project kylo by Teradata.

the class TestSpringConfiguration method connectorPluginManager.

@Bean
public ConnectorPluginManager connectorPluginManager() {
    ConnectorPluginDescriptor descr = new ConnectorPluginDescriptor("dummy", "Dummy Connector", "jdbc");
    ConnectorPlugin plugin = Mockito.mock(ConnectorPlugin.class);
    Mockito.when(plugin.getDescriptor()).thenReturn(descr);
    ConnectorPluginManager mgr = Mockito.mock(ConnectorPluginManager.class);
    Mockito.when(mgr.getPlugin(anyString())).thenReturn(Optional.of(plugin));
    return mgr;
}

Example 3 with ConnectorPluginDescriptor

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor in project kylo by Teradata.

the class DataSourceProvider method updateDataSource.

/**
 * Updates a data source using the specified template.
 *
 * @throws CatalogException if the data source is not valid
 */
@Nonnull
public DataSource updateDataSource(@Nonnull final DataSource source) {
    return metadataService.commit(() -> {
        // Find data source
        final Optional<com.thinkbiganalytics.metadata.api.catalog.DataSource.ID> domainId = Optional.ofNullable(source.getId()).map(metadataProvider::resolveId);
        final com.thinkbiganalytics.metadata.api.catalog.DataSource domain = domainId.flatMap(metadataProvider::find).orElseThrow(() -> new CatalogException("catalog.datasource.notFound.id", source.getId()));
        // Create or update controller service
        final ConnectorPluginDescriptor plugin = pluginManager.getPlugin(domain.getConnector().getPluginId()).map(ConnectorPlugin::getDescriptor).orElse(null);
        if (plugin != null && plugin.getNifiControllerService() != null) {
            createOrUpdateNiFiControllerService(source, plugin, false);
        }
        // Update catalog
        final DataSource updatedDataSource = this.credentialManager.applyPlaceholders(source, SecurityContextUtil.getCurrentPrincipals());
        modelTransform.updateDataSource(updatedDataSource, domain);
        // Return a copy with the connector
        return modelTransform.dataSourceToRestModel().apply(domain);
    });
}

Example 4 with ConnectorPluginDescriptor

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor in project kylo by Teradata.

the class DataSourceProvider method createOrUpdateNiFiControllerService.

/**
 * Creates or updates the NiFi controller service linked to the specified data source.
 */
private void createOrUpdateNiFiControllerService(@Nonnull final DataSource dataSource, @Nonnull final ConnectorPluginDescriptor connectorPluginDescriptor, boolean checkExisting) throws PotentialControllerServiceConflictException {
    ConnectorPluginNiFiControllerService plugin = connectorPluginDescriptor.getNifiControllerService();
    // Resolve properties
    final PropertyPlaceholderHelper.PlaceholderResolver resolver = new DataSourcePlaceholderResolver(dataSource);
    final Map<String, String> properties = new HashMap<>(plugin.getProperties().size());
    final Map<String, ConnectorPluginNiFiControllerServicePropertyDescriptor> propertyDescriptors = plugin.getPropertyDescriptors();
    plugin.getProperties().forEach((key, value) -> {
        final String resolvedValue = placeholderHelper.replacePlaceholders(value, resolver);
        // set empty string to null
        ConnectorPluginNiFiControllerServicePropertyDescriptor descriptor = propertyDescriptors != null ? propertyDescriptors.get(key) : null;
        if (StringUtils.isBlank(resolvedValue) && (descriptor == null || (descriptor != null && !descriptor.isEmptyStringIfNull()))) {
            // set the value to null if its not explicitly configured to be set to an empty string
            properties.put(key, null);
        } else if (resolvedValue != null && !resolvedValue.startsWith("{cipher}")) {
            properties.put(key, resolvedValue);
        }
    });
    // Update or create the controller service
    ControllerServiceDTO controllerService = null;
    if (dataSource.getNifiControllerServiceId() != null && dataSource.getId() != null) {
        controllerService = new ControllerServiceDTO();
        controllerService.setId(dataSource.getNifiControllerServiceId());
        controllerService.setName(dataSource.getTitle());
        controllerService.setProperties(properties);
        try {
            controllerService = nifiRestClient.controllerServices().updateServiceAndReferencingComponents(controllerService);
            dataSource.setNifiControllerServiceId(controllerService.getId());
        } catch (final NifiComponentNotFoundException e) {
            log.warn("Controller service is missing for data source: {}", dataSource.getId(), e);
            controllerService = null;
        }
    }
    if (controllerService == null && StringUtils.isBlank(dataSource.getNifiControllerServiceId())) {
        if (checkExisting) {
            List<ControllerServiceDTO> matchingServices = findMatchingControllerService(dataSource, properties, connectorPluginDescriptor);
            if (matchingServices != null && !matchingServices.isEmpty()) {
                Map<String, String> identityProperties = plugin.getIdentityProperties().stream().collect(Collectors.toMap(propertyKey -> propertyKey, propertyKey -> properties.get(propertyKey)));
                throw new PotentialControllerServiceConflictException(new ControllerServiceConflictEntity(dataSource.getTitle(), identityProperties, matchingServices));
            }
        }
        controllerService = new ControllerServiceDTO();
        controllerService.setType(plugin.getType());
        controllerService.setName(dataSource.getTitle());
        controllerService.setProperties(properties);
        controllerService = nifiRestClient.controllerServices().create(controllerService);
        try {
            nifiRestClient.controllerServices().updateStateById(controllerService.getId(), NiFiControllerServicesRestClient.State.ENABLED);
        } catch (final NifiClientRuntimeException e) {
            log.error("Failed to enable controller service for data source: {}", dataSource.getId(), e);
            nifiRestClient.controllerServices().disableAndDeleteAsync(controllerService.getId());
            throw e;
        }
        dataSource.setNifiControllerServiceId(controllerService.getId());
    }
}

Example 5 with ConnectorPluginDescriptor

use of com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor in project kylo by Teradata.

the class DataSourceProvider method createDataSource.

/**
 * Creates a new data source using the specified template.
 *
 * @throws CatalogException if the data source is not valid
 */
@Nonnull
public DataSource createDataSource(@Nonnull final DataSource source, boolean checkExisting) throws PotentialControllerServiceConflictException {
    return metadataService.commit(() -> {
        // Find connector
        final com.thinkbiganalytics.metadata.api.catalog.Connector connector = Optional.ofNullable(source.getConnector()).map(Connector::getId).map(connectorProvider::resolveId).flatMap(connectorProvider::find).orElseThrow(() -> new CatalogException("catalog.datasource.connector.invalid"));
        // Create data source
        final com.thinkbiganalytics.metadata.api.catalog.DataSource domain = metadataProvider.create(connector.getId(), source.getTitle());
        // Create or update controller service
        final ConnectorPluginDescriptor plugin = pluginManager.getPlugin(connector.getPluginId()).map(ConnectorPlugin::getDescriptor).orElse(null);
        if (plugin != null && plugin.getNifiControllerService() != null) {
            createOrUpdateNiFiControllerService(source, plugin, checkExisting);
        }
        // Update catalog
        final DataSource updatedDataSource = this.credentialManager.applyPlaceholders(source, SecurityContextUtil.getCurrentPrincipals());
        modelTransform.updateDataSource(updatedDataSource, domain);
        // Return a copy with the connector
        return modelTransform.dataSourceToRestModel().apply(domain);
    });
}