Search in sources :

Example 1 with DataSourceProvider

use of com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider in project kylo by Teradata.

the class SparkShellProxyController method resolveDatasources.

/**
 * Retrieves all details of the specified data sources.
 */
@Nonnull
private List<Datasource> resolveDatasources(@Nonnull final List<Datasource> sources) {
    // Verify access to data sources
    accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
    final List<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> datasourceIds = metadata.read(() -> sources.stream().map(com.thinkbiganalytics.metadata.datasource.Datasource::getId).map(datasourceProvider::resolve).map(id -> {
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasourceProvider.getDatasource(id);
        if (datasource != null) {
            return datasource.getId();
        } else {
            throw new BadRequestException("No datasource exists with the given ID: " + id);
        }
    }).collect(Collectors.toList()));
    // Retrieve admin-level details
    return metadata.read(() -> datasourceIds.stream().map(datasourceProvider::getDatasource).map(datasource -> {
        if (datasource instanceof com.thinkbiganalytics.metadata.api.datasource.UserDatasource) {
            return (com.thinkbiganalytics.metadata.datasource.Datasource) datasourceTransform.toDatasource(datasource, DatasourceModelTransform.Level.ADMIN);
        } else {
            throw new BadRequestException("Not a supported datasource: " + datasource.getClass().getSimpleName() + " " + datasource.getId());
        }
    }).map(datasource -> {
        if (datasource instanceof com.thinkbiganalytics.metadata.datasource.JdbcDatasource) {
            return new JdbcDatasource((com.thinkbiganalytics.metadata.datasource.JdbcDatasource) datasource);
        } else {
            throw new BadRequestException("Not a supported datasource: " + datasource.getClass().getSimpleName());
        }
    }).collect(Collectors.toList()), MetadataAccess.SERVICE);
}
Also used : Datasource(com.thinkbiganalytics.spark.rest.model.Datasource) JdbcDatasource(com.thinkbiganalytics.spark.rest.model.JdbcDatasource) Produces(javax.ws.rs.Produces) DataSourceProvider(com.thinkbiganalytics.metadata.api.catalog.DataSourceProvider) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Autowired(org.springframework.beans.factory.annotation.Autowired) ApiParam(io.swagger.annotations.ApiParam) AsyncResponseSubscriber(com.thinkbiganalytics.kylo.reactive.AsyncResponseSubscriber) StringUtils(org.apache.commons.lang3.StringUtils) PreviewDataSetTransformResponse(com.thinkbiganalytics.spark.rest.model.PreviewDataSetTransformResponse) MediaType(javax.ws.rs.core.MediaType) SaveRequest(com.thinkbiganalytics.spark.rest.model.SaveRequest) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) AccessController(com.thinkbiganalytics.security.AccessController) PageSpec(com.thinkbiganalytics.spark.rest.model.PageSpec) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) SecurityContextHolder(org.springframework.security.core.context.SecurityContextHolder) SchemaParserAnnotationTransformer(com.thinkbiganalytics.discovery.rest.controller.SchemaParserAnnotationTransformer) DataSet(com.thinkbiganalytics.kylo.catalog.rest.model.DataSet) AsyncResponse(javax.ws.rs.container.AsyncResponse) Set(java.util.Set) FileParserFactory(com.thinkbiganalytics.discovery.FileParserFactory) SparkJobContext(com.thinkbiganalytics.kylo.spark.job.SparkJobContext) Suspended(javax.ws.rs.container.Suspended) SparkException(com.thinkbiganalytics.kylo.spark.SparkException) Stream(java.util.stream.Stream) ServerStatusResponse(com.thinkbiganalytics.spark.rest.model.ServerStatusResponse) KyloCatalogReadRequest(com.thinkbiganalytics.spark.rest.model.KyloCatalogReadRequest) WebApplicationException(javax.ws.rs.WebApplicationException) GET(javax.ws.rs.GET) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) SparkJobResponse(com.thinkbiganalytics.kylo.spark.rest.model.job.SparkJobResponse) FileSchemaParser(com.thinkbiganalytics.discovery.parser.FileSchemaParser) HttpServletRequest(javax.servlet.http.HttpServletRequest) RestResponseStatus(com.thinkbiganalytics.rest.model.RestResponseStatus) Datasource(com.thinkbiganalytics.spark.rest.model.Datasource) Api(io.swagger.annotations.Api) Nullable(javax.annotation.Nullable) SparkJobService(com.thinkbiganalytics.kylo.spark.job.SparkJobService) MessageSource(org.springframework.context.MessageSource) SchemaParserDescriptorUtil(com.thinkbiganalytics.discovery.rest.controller.SchemaParserDescriptorUtil) FileMetadataCompletionTask(com.thinkbiganalytics.spark.rest.filemetadata.tasks.FileMetadataCompletionTask) SparkShellProcess(com.thinkbiganalytics.spark.shell.SparkShellProcess) JdbcDatasource(com.thinkbiganalytics.spark.rest.model.JdbcDatasource) SparkJobRequest(com.thinkbiganalytics.kylo.spark.rest.model.job.SparkJobRequest) FileMetadataScalaScriptGenerator(com.thinkbiganalytics.kylo.spark.file.metadata.FileMetadataScalaScriptGenerator) SparkJobStatus(com.thinkbiganalytics.kylo.spark.job.SparkJobStatus) SwaggerDefinition(io.swagger.annotations.SwaggerDefinition) ApiResponse(io.swagger.annotations.ApiResponse) RegistrationRequest(com.thinkbiganalytics.spark.rest.model.RegistrationRequest) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) TransformRequest(com.thinkbiganalytics.spark.rest.model.TransformRequest) NoSuchMessageException(org.springframework.context.NoSuchMessageException) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) User(org.springframework.security.core.userdetails.User) TimeoutException(java.util.concurrent.TimeoutException) DefaultDataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate) RequestContextUtils(org.springframework.web.servlet.support.RequestContextUtils) ApiOperation(io.swagger.annotations.ApiOperation) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) Consumes(javax.ws.rs.Consumes) Locale(java.util.Locale) TransformResultModifier(com.thinkbiganalytics.spark.rest.model.TransformResultModifier) BadRequestException(javax.ws.rs.BadRequestException) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) CatalogModelTransform(com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform) SparkShellRestClient(com.thinkbiganalytics.spark.shell.SparkShellRestClient) FileMetadataTransformResponseModifier(com.thinkbiganalytics.spark.rest.filemetadata.FileMetadataTransformResponseModifier) SchemaParserDescriptor(com.thinkbiganalytics.discovery.model.SchemaParserDescriptor) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) DataSources(com.thinkbiganalytics.spark.rest.model.DataSources) Objects(java.util.Objects) List(java.util.List) HttpHeaders(javax.ws.rs.core.HttpHeaders) Response(javax.ws.rs.core.Response) SaveResponse(com.thinkbiganalytics.spark.rest.model.SaveResponse) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) Authentication(org.springframework.security.core.Authentication) PathParam(javax.ws.rs.PathParam) DataSourceUtil(com.thinkbiganalytics.kylo.catalog.datasource.DataSourceUtil) ApiResponses(io.swagger.annotations.ApiResponses) DataSetUtil(com.thinkbiganalytics.kylo.catalog.dataset.DataSetUtil) HashSet(java.util.HashSet) Inject(javax.inject.Inject) TransformResponse(com.thinkbiganalytics.spark.rest.model.TransformResponse) Qualifier(org.springframework.beans.factory.annotation.Qualifier) SubscriberFactory(com.thinkbiganalytics.kylo.reactive.SubscriberFactory) Tag(io.swagger.annotations.Tag) Subscriber(org.reactivestreams.Subscriber) Nonnull(javax.annotation.Nonnull) Logger(org.slf4j.Logger) POST(javax.ws.rs.POST) SparkShellSaveException(com.thinkbiganalytics.spark.shell.SparkShellSaveException) PreviewDataSetRequest(com.thinkbiganalytics.spark.rest.model.PreviewDataSetRequest) FileMetadataTaskService(com.thinkbiganalytics.spark.rest.filemetadata.tasks.FileMetadataTaskService) ModifiedTransformResponse(com.thinkbiganalytics.spark.rest.model.ModifiedTransformResponse) SparkShellProcessManager(com.thinkbiganalytics.spark.shell.SparkShellProcessManager) TimeUnit(java.util.concurrent.TimeUnit) Component(org.springframework.stereotype.Component) SparkShellTransformException(com.thinkbiganalytics.spark.shell.SparkShellTransformException) Collections(java.util.Collections) JdbcDatasource(com.thinkbiganalytics.spark.rest.model.JdbcDatasource) BadRequestException(javax.ws.rs.BadRequestException) Nonnull(javax.annotation.Nonnull)

Aggregations

FileParserFactory (com.thinkbiganalytics.discovery.FileParserFactory)1 SchemaParserDescriptor (com.thinkbiganalytics.discovery.model.SchemaParserDescriptor)1 FileSchemaParser (com.thinkbiganalytics.discovery.parser.FileSchemaParser)1 SchemaParserAnnotationTransformer (com.thinkbiganalytics.discovery.rest.controller.SchemaParserAnnotationTransformer)1 SchemaParserDescriptorUtil (com.thinkbiganalytics.discovery.rest.controller.SchemaParserDescriptorUtil)1 FeedServicesAccessControl (com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl)1 DatasourceModelTransform (com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform)1 DataSetUtil (com.thinkbiganalytics.kylo.catalog.dataset.DataSetUtil)1 DataSourceUtil (com.thinkbiganalytics.kylo.catalog.datasource.DataSourceUtil)1 CatalogModelTransform (com.thinkbiganalytics.kylo.catalog.rest.model.CatalogModelTransform)1 DataSet (com.thinkbiganalytics.kylo.catalog.rest.model.DataSet)1 DataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate)1 DataSource (com.thinkbiganalytics.kylo.catalog.rest.model.DataSource)1 DefaultDataSetTemplate (com.thinkbiganalytics.kylo.catalog.rest.model.DefaultDataSetTemplate)1 AsyncResponseSubscriber (com.thinkbiganalytics.kylo.reactive.AsyncResponseSubscriber)1 SubscriberFactory (com.thinkbiganalytics.kylo.reactive.SubscriberFactory)1 SparkException (com.thinkbiganalytics.kylo.spark.SparkException)1 FileMetadataScalaScriptGenerator (com.thinkbiganalytics.kylo.spark.file.metadata.FileMetadataScalaScriptGenerator)1 SparkJobContext (com.thinkbiganalytics.kylo.spark.job.SparkJobContext)1 SparkJobService (com.thinkbiganalytics.kylo.spark.job.SparkJobService)1