Search in sources :

Example 1 with DatasourceCriteria

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria in project kylo by Teradata.

the class DataSourceProvider method findAllDataSources.

/**
 * Gets the list of available connectors.
 */
@Nonnull
public Page<DataSource> findAllDataSources(@Nonnull final Pageable pageable, @Nullable final String filter) {
    // Filter catalog data sources
    final Supplier<Stream<DataSource>> catalogDataSources = () -> metadataService.read(() -> metadataProvider.findAll().stream().map(modelTransform.dataSourceToRestModel()));
    // Filter feed data sources
    final DatasourceCriteria feedDataSourcesCriteria = feedDataSourceProvider.datasetCriteria().type(UserDatasource.class);
    final Supplier<Stream<DataSource>> feedDataSources = () -> feedDataSourceProvider.getDatasources(feedDataSourcesCriteria).stream().filter(containsIgnoreCase(Datasource::getName, filter)).map(datasource -> toDataSource(datasource, DatasourceModelTransform.Level.BASIC)).filter(Objects::nonNull);
    // Sort and paginate
    final List<DataSource> filteredDataSources = Stream.concat(catalogDataSources.get(), feedDataSources.get()).sorted(Comparator.comparing(DataSource::getTitle)).skip(pageable.getOffset()).limit(pageable.getPageSize()).peek(findConnector()).collect(Collectors.toList());
    return new PageImpl<>(filteredDataSources, pageable, catalogDataSources.get().count() + feedDataSources.get().count());
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Autowired(org.springframework.beans.factory.annotation.Autowired) StringUtils(org.apache.commons.lang3.StringUtils) DefaultDataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate) NifiClientRuntimeException(com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException) DatasourceCriteria(com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Map(java.util.Map) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Pageable(org.springframework.data.domain.Pageable) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) NiFiControllerServicesRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiControllerServicesRestClient) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) PropertyPlaceholderHelper(org.springframework.util.PropertyPlaceholderHelper) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) Predicate(java.util.function.Predicate) Page(org.springframework.data.domain.Page) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) ConnectorPluginNiFiControllerService(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService) Optional(java.util.Optional) PageImpl(org.springframework.data.domain.PageImpl) ConnectorPlugin(com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin) ConnectorPluginManager(com.thinkbiganalytics.kylo.catalog.ConnectorPluginManager) HashMap(java.util.HashMap) Function(java.util.function.Function) Supplier(java.util.function.Supplier) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) ConnectorPluginNiFiControllerServicePropertyDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) CatalogException(com.thinkbiganalytics.kylo.catalog.CatalogException) DataSourceCredentialManager(com.thinkbiganalytics.kylo.catalog.credential.api.DataSourceCredentialManager) NifiComponentNotFoundException(com.thinkbiganalytics.nifi.rest.client.NifiComponentNotFoundException) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ConnectorProvider(com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider) UserDatasource(com.thinkbiganalytics.metadata.api.datasource.UserDatasource) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) SecurityContextUtil(com.thinkbiganalytics.security.context.SecurityContextUtil) Logger(org.slf4j.Logger) Consumer(java.util.function.Consumer) Component(org.springframework.stereotype.Component) Connector(com.thinkbiganalytics.kylo.catalog.rest.model.Connector) ConnectorPluginDescriptor(com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor) Comparator(java.util.Comparator) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) PageImpl(org.springframework.data.domain.PageImpl) Objects(java.util.Objects) DatasourceCriteria(com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria) Stream(java.util.stream.Stream) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Nonnull(javax.annotation.Nonnull)

Aggregations

DatasourceModelTransform (com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform)1 CatalogException (com.thinkbiganalytics.kylo.catalog.CatalogException)1 ConnectorPluginManager (com.thinkbiganalytics.kylo.catalog.ConnectorPluginManager)1 DataSourceCredentialManager (com.thinkbiganalytics.kylo.catalog.credential.api.DataSourceCredentialManager)1 CatalogModelTransform (com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform)1 Connector (com.thinkbiganalytics.kylo.catalog.rest.model.Connector)1 ConnectorPluginDescriptor (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginDescriptor)1 ConnectorPluginNiFiControllerService (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerService)1 ConnectorPluginNiFiControllerServicePropertyDescriptor (com.thinkbiganalytics.kylo.catalog.rest.model.ConnectorPluginNiFiControllerServicePropertyDescriptor)1 DataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate)1 DataSource (com.thinkbiganalytics.kylo.catalog.rest.model.DataSource)1 DefaultDataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate)1 ConnectorPlugin (com.thinkbiganalytics.kylo.catalog.spi.ConnectorPlugin)1 MetadataAccess (com.thinkbiganalytics.metadata.api.MetadataAccess)1 ConnectorProvider (com.thinkbiganalytics.metadata.api.catalog.ConnectorProvider)1 Datasource (com.thinkbiganalytics.metadata.api.datasource.Datasource)1 DatasourceCriteria (com.thinkbiganalytics.metadata.api.datasource.DatasourceCriteria)1 DatasourceProvider (com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider)1 UserDatasource (com.thinkbiganalytics.metadata.api.datasource.UserDatasource)1 JdbcDatasource (com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource)1