Search in sources :

Example 1 with DatasourceProvider

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider in project kylo by Teradata.

the class DatasourceController method previewQuery.

/**
 * Executes a query on the specified datasource.
 *
 * @param idStr the datasource id
 * @param query the SQL query
 * @return the SQL result
 */
@POST
@Path("{id}/preview/{schema}/{table}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Executes a preview query appropriate for the type of datasource and returns the result.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the result.", response = QueryResult.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 400, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response previewQuery(@PathParam("id") final String idStr, @PathParam("schema") final String schema, @PathParam("table") final String tableName, @QueryParam("limit") @DefaultValue("10") final int limit) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Execute query
    return metadata.read(() -> {
        final QueryResult result = id.map(datasourceProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).map(datasource -> dbcpConnectionPoolTableInfo.executePreviewQueryForDatasource(datasource, schema, tableName, limit)).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        return Response.ok(result).build();
    }, MetadataAccess.SERVICE);
}
Also used : JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) Arrays(java.util.Arrays) JdbcDatasourceDetails(com.thinkbiganalytics.metadata.api.datasource.JdbcDatasourceDetails) Produces(javax.ws.rs.Produces) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Collections2(com.google.common.collect.Collections2) DatasourceDefinition(com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinition) StringUtils(org.apache.commons.lang3.StringUtils) Model(com.thinkbiganalytics.feedmgr.rest.Model) DatasourceDefinitionProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinitionProvider) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) Consumes(javax.ws.rs.Consumes) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) DefaultValue(javax.ws.rs.DefaultValue) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Formatters(com.thinkbiganalytics.Formatters) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) DELETE(javax.ws.rs.DELETE) PermissionsChange(com.thinkbiganalytics.security.rest.model.PermissionsChange) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) Set(java.util.Set) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) List(java.util.List) Principal(java.security.Principal) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) AccessControlException(java.security.AccessControlException) Optional(java.util.Optional) WebApplicationException(javax.ws.rs.WebApplicationException) RoleMembershipChange(com.thinkbiganalytics.security.rest.model.RoleMembershipChange) PathParam(javax.ws.rs.PathParam) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) GET(javax.ws.rs.GET) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) RestResponseStatus(com.thinkbiganalytics.rest.model.RestResponseStatus) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Api(io.swagger.annotations.Api) Tag(io.swagger.annotations.Tag) Nonnull(javax.annotation.Nonnull) Logger(org.slf4j.Logger) CannotGetJdbcConnectionException(org.springframework.jdbc.CannotGetJdbcConnectionException) POST(javax.ws.rs.POST) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) AccessControlled(com.thinkbiganalytics.security.AccessControlled) DBCPConnectionPoolService(com.thinkbiganalytics.feedmgr.nifi.controllerservice.DBCPConnectionPoolService) Component(org.springframework.stereotype.Component) SwaggerDefinition(io.swagger.annotations.SwaggerDefinition) DatasourceAccessControl(com.thinkbiganalytics.metadata.api.datasource.security.DatasourceAccessControl) ApiResponse(io.swagger.annotations.ApiResponse) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) DatasourceCriteria(com.thinkbiganalytics.metadata.rest.model.data.DatasourceCriteria) QueryResult(com.thinkbiganalytics.discovery.schema.QueryResult) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) NotFoundException(javax.ws.rs.NotFoundException) QueryResult(com.thinkbiganalytics.discovery.schema.QueryResult) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) Produces(javax.ws.rs.Produces) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 2 with DatasourceProvider

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider in project kylo by Teradata.

the class DatasourceController method describeTable.

/**
 * Gets the schema of the specified table using the specified data source.
 *
 * @param idStr     the data source id
 * @param tableName the table name
 * @param schema    the schema name, or {@code null} to search all schemas
 * @return the table and field details
 */
@GET
@Path("{id}/tables/{tableName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the schema of the specified table.", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table schema.", response = TableSchema.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response describeTable(@PathParam("id") final String idStr, @PathParam("tableName") final String tableName, @QueryParam("schema") final String schema) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Retrieve table description using system user
    return metadata.read(() -> {
        final TableSchema tableSchema = id.map(datasourceProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).map(datasource -> dbcpConnectionPoolTableInfo.describeTableForDatasource(datasource, schema, tableName)).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        return Response.ok(tableSchema).build();
    }, MetadataAccess.SERVICE);
}
Also used : JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) Arrays(java.util.Arrays) JdbcDatasourceDetails(com.thinkbiganalytics.metadata.api.datasource.JdbcDatasourceDetails) Produces(javax.ws.rs.Produces) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Collections2(com.google.common.collect.Collections2) DatasourceDefinition(com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinition) StringUtils(org.apache.commons.lang3.StringUtils) Model(com.thinkbiganalytics.feedmgr.rest.Model) DatasourceDefinitionProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinitionProvider) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) Consumes(javax.ws.rs.Consumes) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) DefaultValue(javax.ws.rs.DefaultValue) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Formatters(com.thinkbiganalytics.Formatters) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) DELETE(javax.ws.rs.DELETE) PermissionsChange(com.thinkbiganalytics.security.rest.model.PermissionsChange) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) Set(java.util.Set) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) List(java.util.List) Principal(java.security.Principal) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) AccessControlException(java.security.AccessControlException) Optional(java.util.Optional) WebApplicationException(javax.ws.rs.WebApplicationException) RoleMembershipChange(com.thinkbiganalytics.security.rest.model.RoleMembershipChange) PathParam(javax.ws.rs.PathParam) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) GET(javax.ws.rs.GET) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) RestResponseStatus(com.thinkbiganalytics.rest.model.RestResponseStatus) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Api(io.swagger.annotations.Api) Tag(io.swagger.annotations.Tag) Nonnull(javax.annotation.Nonnull) Logger(org.slf4j.Logger) CannotGetJdbcConnectionException(org.springframework.jdbc.CannotGetJdbcConnectionException) POST(javax.ws.rs.POST) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) AccessControlled(com.thinkbiganalytics.security.AccessControlled) DBCPConnectionPoolService(com.thinkbiganalytics.feedmgr.nifi.controllerservice.DBCPConnectionPoolService) Component(org.springframework.stereotype.Component) SwaggerDefinition(io.swagger.annotations.SwaggerDefinition) DatasourceAccessControl(com.thinkbiganalytics.metadata.api.datasource.security.DatasourceAccessControl) ApiResponse(io.swagger.annotations.ApiResponse) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) DatasourceCriteria(com.thinkbiganalytics.metadata.rest.model.data.DatasourceCriteria) QueryResult(com.thinkbiganalytics.discovery.schema.QueryResult) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) NotFoundException(javax.ws.rs.NotFoundException) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 3 with DatasourceProvider

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider in project kylo by Teradata.

the class DatasourceController method getTablesAndColumns.

/**
 * Gets the tables and their columns from the specified data source for given schema
 *
 * @param idStr  the data source id
 * @param schema the schema name, or {@code null} for all schemas
 * @return the list of tables and their columns
 */
@GET
@Path("{id}/table-columns")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the tables and their columns from the data source for given schema", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns tables and columns", response = String.class, responseContainer = "List"), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response getTablesAndColumns(@PathParam("id") final String idStr, @QueryParam("schema") final String schema) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Retrieve table names using system user
    return metadata.read(() -> {
        JdbcDatasource datasource = id.map(datasourceProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        List<String> tableNamesForDatasource = dbcpConnectionPoolTableInfo.getTableNamesForDatasource(datasource, schema, null);
        List<DatabaseMetadata> tables = new ArrayList<>();
        if (tableNamesForDatasource != null) {
            tables = tableNamesForDatasource.stream().flatMap(schemaNameDotTableName -> {
                String tableName = schemaNameDotTableName.substring(schema.length() + 1);
                TableSchema tableSchema = dbcpConnectionPoolTableInfo.describeTableForDatasource(datasource, schema, tableName);
                return tableSchema.getFields().stream().map(field -> {
                    DefaultDatabaseMetadata meta = new DefaultDatabaseMetadata();
                    meta.setDatabaseName(schema);
                    meta.setColumnName(field.getName());
                    meta.setTableName(tableName);
                    return meta;
                });
            }).collect(Collectors.toList());
        }
        return Response.ok(tables).build();
    }, MetadataAccess.SERVICE);
}
Also used : JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) Arrays(java.util.Arrays) JdbcDatasourceDetails(com.thinkbiganalytics.metadata.api.datasource.JdbcDatasourceDetails) Produces(javax.ws.rs.Produces) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Collections2(com.google.common.collect.Collections2) DatasourceDefinition(com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinition) StringUtils(org.apache.commons.lang3.StringUtils) Model(com.thinkbiganalytics.feedmgr.rest.Model) DatasourceDefinitionProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinitionProvider) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) Consumes(javax.ws.rs.Consumes) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) DefaultValue(javax.ws.rs.DefaultValue) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Formatters(com.thinkbiganalytics.Formatters) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) DELETE(javax.ws.rs.DELETE) PermissionsChange(com.thinkbiganalytics.security.rest.model.PermissionsChange) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) Set(java.util.Set) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) List(java.util.List) Principal(java.security.Principal) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) AccessControlException(java.security.AccessControlException) Optional(java.util.Optional) WebApplicationException(javax.ws.rs.WebApplicationException) RoleMembershipChange(com.thinkbiganalytics.security.rest.model.RoleMembershipChange) PathParam(javax.ws.rs.PathParam) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) GET(javax.ws.rs.GET) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) RestResponseStatus(com.thinkbiganalytics.rest.model.RestResponseStatus) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Api(io.swagger.annotations.Api) Tag(io.swagger.annotations.Tag) Nonnull(javax.annotation.Nonnull) Logger(org.slf4j.Logger) CannotGetJdbcConnectionException(org.springframework.jdbc.CannotGetJdbcConnectionException) POST(javax.ws.rs.POST) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) AccessControlled(com.thinkbiganalytics.security.AccessControlled) DBCPConnectionPoolService(com.thinkbiganalytics.feedmgr.nifi.controllerservice.DBCPConnectionPoolService) Component(org.springframework.stereotype.Component) SwaggerDefinition(io.swagger.annotations.SwaggerDefinition) DatasourceAccessControl(com.thinkbiganalytics.metadata.api.datasource.security.DatasourceAccessControl) ApiResponse(io.swagger.annotations.ApiResponse) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) DatasourceCriteria(com.thinkbiganalytics.metadata.rest.model.data.DatasourceCriteria) QueryResult(com.thinkbiganalytics.discovery.schema.QueryResult) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) ArrayList(java.util.ArrayList) NotFoundException(javax.ws.rs.NotFoundException) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 4 with DatasourceProvider

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider in project kylo by Teradata.

the class DerivedDatasourceFactory method ensureDataTransformationSourceDatasources.

/**
 * Builds the list of data sources for the specified data transformation feed.
 *
 * @param feed the feed
 * @return the list of data sources
 * @throws NullPointerException if the feed has no data transformation
 */
@Nonnull
private Set<Datasource.ID> ensureDataTransformationSourceDatasources(@Nonnull final FeedMetadata feed) {
    final Set<String> dataSetIds = new HashSet<>();
    final Set<Datasource.ID> datasources = new HashSet<>();
    if (feed.getSourceDataSets() != null) {
        List<String> datasetIds = feed.getSourceDataSets().stream().map(ds -> ds.getId()).collect(Collectors.toList());
        dataSetIds.addAll(datasetIds);
    }
    final List<String> catalogSources = feed.getDataTransformation().getCatalogDataSourceIds() != null ? feed.getDataTransformation().getCatalogDataSourceIds() : new ArrayList<>();
    // Extract nodes in chart view model
    @SuppressWarnings("unchecked") final Stream<Map<String, Object>> nodes = Optional.ofNullable(feed.getDataTransformation().getChartViewModel()).map(model -> (List<Map<String, Object>>) model.get("nodes")).map(Collection::stream).orElse(Stream.empty());
    // Create a data source for each node
    final DatasourceDefinition hiveDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_HIVE_DEFINITION);
    final DatasourceDefinition jdbcDefinition = datasourceDefinitionProvider.findByProcessorType(DATA_TRANSFORMATION_JDBC_DEFINITION);
    nodes.forEach(node -> {
        // Filter data sets
        if (!StringUtils.equalsAnyIgnoreCase((String) node.get("datasourceId"), null, "HIVE") && node.get("dataset") != null && !Objects.equals(node.get("datasetMatchesUserDataSource"), Boolean.TRUE)) {
            dataSetIds.add((String) node.get("datasourceId"));
            return;
        }
        // Extract properties from node
        DatasourceDefinition datasourceDefinition = null;
        final Map<String, String> properties = new HashMap<>();
        String userDatasourceId = (String) node.get("datasourceId");
        if ((userDatasourceId == null && node.get("dataset") == null) || (userDatasourceId != null && userDatasourceId.equalsIgnoreCase("HIVE"))) {
            final String name = (String) node.get("name");
            datasourceDefinition = hiveDefinition;
            properties.put(HIVE_SCHEMA_KEY, StringUtils.trim(StringUtils.substringBefore(name, ".")));
            properties.put(HIVE_TABLE_KEY, StringUtils.trim(StringUtils.substringAfterLast(name, ".")));
        } else if (userDatasourceId != null) {
            final Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(userDatasourceId));
            if (datasource != null) {
                datasourceDefinition = jdbcDefinition;
                properties.put(JDBC_CONNECTION_KEY, datasource.getName());
                properties.put(JDBC_TABLE_KEY, (String) node.get("name"));
                properties.putAll(parseDataTransformControllerServiceProperties(datasourceDefinition, datasource.getName()));
            }
        }
        if (datasourceDefinition != null) {
            // Create the derived data source
            final String identityString = propertyExpressionResolver.resolveVariables(datasourceDefinition.getIdentityString(), properties);
            final String title = datasourceDefinition.getTitle() != null ? propertyExpressionResolver.resolveVariables(datasourceDefinition.getTitle(), properties) : identityString;
            final String desc = propertyExpressionResolver.resolveVariables(datasourceDefinition.getDescription(), properties);
            final DerivedDatasource datasource = datasourceProvider.ensureDerivedDatasource(datasourceDefinition.getDatasourceType(), identityString, title, desc, new HashMap<>(properties));
            datasources.add(datasource.getId());
        }
    });
    // Build the data sources from the data source ids
    if (dataSetIds.isEmpty()) {
        final List<String> datasourceIds = Optional.ofNullable(feed.getDataTransformation()).map(FeedDataTransformation::getDatasourceIds).orElse(Collections.emptyList());
        datasourceIds.stream().filter(id -> !dataSetIds.contains(id) && !catalogSources.contains(id)).map(datasourceProvider::resolve).forEach(datasources::add);
    }
    return datasources;
}
Also used : FeedDataTransformation(com.thinkbiganalytics.feedmgr.rest.model.FeedDataTransformation) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) FeedMetadata(com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata) StringUtils(org.apache.commons.lang3.StringUtils) DatasourceDefinitionProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinitionProvider) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) DatasourceDefinition(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition) TemplateProcessorDatasourceDefinition(com.thinkbiganalytics.feedmgr.rest.model.TemplateProcessorDatasourceDefinition) Map(java.util.Map) PropertyExpressionResolver(com.thinkbiganalytics.feedmgr.nifi.PropertyExpressionResolver) TableSetup(com.thinkbiganalytics.feedmgr.rest.model.schema.TableSetup) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) RegisteredTemplate(com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate) Nonnull(javax.annotation.Nonnull) FeedManagerTemplateService(com.thinkbiganalytics.feedmgr.service.template.FeedManagerTemplateService) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) Logger(org.slf4j.Logger) ControllerServiceDTO(org.apache.nifi.web.api.dto.ControllerServiceDTO) NifiProperty(com.thinkbiganalytics.nifi.rest.model.NifiProperty) NifiControllerServiceProperties(com.thinkbiganalytics.feedmgr.nifi.NifiControllerServiceProperties) Collection(java.util.Collection) Set(java.util.Set) Collectors(java.util.stream.Collectors) Serializable(java.io.Serializable) Objects(java.util.Objects) RegisteredTemplateCache(com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateCache) List(java.util.List) Stream(java.util.stream.Stream) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) Optional(java.util.Optional) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) Collections(java.util.Collections) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) Datasource(com.thinkbiganalytics.metadata.api.datasource.Datasource) HashMap(java.util.HashMap) DerivedDatasource(com.thinkbiganalytics.metadata.api.datasource.DerivedDatasource) DatasourceDefinition(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinition) TemplateProcessorDatasourceDefinition(com.thinkbiganalytics.feedmgr.rest.model.TemplateProcessorDatasourceDefinition) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet) Nonnull(javax.annotation.Nonnull)

Example 5 with DatasourceProvider

use of com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider in project kylo by Teradata.

the class DatasourceController method getTableNames.

/**
 * Gets the table names from the specified data source.
 *
 * @param idStr  the data source id
 * @param schema the schema name, or {@code null} for all schemas
 * @return the list of table names
 */
@GET
@Path("{id}/tables")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the table names from the data source.", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the table names.", response = String.class, responseContainer = "List"), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response getTableNames(@PathParam("id") final String idStr, @QueryParam("schema") final String schema, @QueryParam("tableName") final String tableName) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasourceProvider.getDatasource(datasourceProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Retrieve table names using system user
    return metadata.read(() -> {
        final List<String> tables = id.map(datasourceProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).map(datasource -> dbcpConnectionPoolTableInfo.getTableNamesForDatasource(datasource, schema, tableName)).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        return Response.ok(tables).build();
    }, MetadataAccess.SERVICE);
}
Also used : JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) Arrays(java.util.Arrays) JdbcDatasourceDetails(com.thinkbiganalytics.metadata.api.datasource.JdbcDatasourceDetails) Produces(javax.ws.rs.Produces) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) LoggerFactory(org.slf4j.LoggerFactory) Path(javax.ws.rs.Path) DatasourceModelTransform(com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform) Collections2(com.google.common.collect.Collections2) DatasourceDefinition(com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinition) StringUtils(org.apache.commons.lang3.StringUtils) Model(com.thinkbiganalytics.feedmgr.rest.Model) DatasourceDefinitionProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceDefinitionProvider) ApiOperation(io.swagger.annotations.ApiOperation) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) DataSetProvider(com.thinkbiganalytics.metadata.api.catalog.DataSetProvider) Consumes(javax.ws.rs.Consumes) Map(java.util.Map) AccessController(com.thinkbiganalytics.security.AccessController) DefaultValue(javax.ws.rs.DefaultValue) NiFiRestClient(com.thinkbiganalytics.nifi.rest.client.NiFiRestClient) Formatters(com.thinkbiganalytics.Formatters) FeedServicesAccessControl(com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) DELETE(javax.ws.rs.DELETE) PermissionsChange(com.thinkbiganalytics.security.rest.model.PermissionsChange) ActionGroup(com.thinkbiganalytics.security.rest.model.ActionGroup) Set(java.util.Set) Collectors(java.util.stream.Collectors) NotFoundException(javax.ws.rs.NotFoundException) List(java.util.List) Principal(java.security.Principal) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) AccessControlException(java.security.AccessControlException) Optional(java.util.Optional) WebApplicationException(javax.ws.rs.WebApplicationException) RoleMembershipChange(com.thinkbiganalytics.security.rest.model.RoleMembershipChange) PathParam(javax.ws.rs.PathParam) SecurityModelTransform(com.thinkbiganalytics.security.rest.controller.SecurityModelTransform) GET(javax.ws.rs.GET) DataSet(com.thinkbiganalytics.metadata.api.catalog.DataSet) HashMap(java.util.HashMap) ApiResponses(io.swagger.annotations.ApiResponses) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Inject(javax.inject.Inject) RestResponseStatus(com.thinkbiganalytics.rest.model.RestResponseStatus) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) UserDatasource(com.thinkbiganalytics.metadata.rest.model.data.UserDatasource) Api(io.swagger.annotations.Api) Tag(io.swagger.annotations.Tag) Nonnull(javax.annotation.Nonnull) Logger(org.slf4j.Logger) CannotGetJdbcConnectionException(org.springframework.jdbc.CannotGetJdbcConnectionException) POST(javax.ws.rs.POST) SecurityService(com.thinkbiganalytics.feedmgr.service.security.SecurityService) Datasource(com.thinkbiganalytics.metadata.rest.model.data.Datasource) AccessControlled(com.thinkbiganalytics.security.AccessControlled) DBCPConnectionPoolService(com.thinkbiganalytics.feedmgr.nifi.controllerservice.DBCPConnectionPoolService) Component(org.springframework.stereotype.Component) SwaggerDefinition(io.swagger.annotations.SwaggerDefinition) DatasourceAccessControl(com.thinkbiganalytics.metadata.api.datasource.security.DatasourceAccessControl) ApiResponse(io.swagger.annotations.ApiResponse) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DatasourceProvider(com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider) DatasourceCriteria(com.thinkbiganalytics.metadata.rest.model.data.DatasourceCriteria) QueryResult(com.thinkbiganalytics.discovery.schema.QueryResult) JdbcDatasource(com.thinkbiganalytics.metadata.rest.model.data.JdbcDatasource) NotFoundException(javax.ws.rs.NotFoundException) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Aggregations

MetadataAccess (com.thinkbiganalytics.metadata.api.MetadataAccess)10 DatasourceProvider (com.thinkbiganalytics.metadata.api.datasource.DatasourceProvider)10 List (java.util.List)10 Optional (java.util.Optional)10 Collectors (java.util.stream.Collectors)10 Inject (javax.inject.Inject)10 FeedServicesAccessControl (com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl)9 DatasourceModelTransform (com.thinkbiganalytics.feedmgr.service.datasource.DatasourceModelTransform)9 DataSetProvider (com.thinkbiganalytics.metadata.api.catalog.DataSetProvider)9 ArrayList (java.util.ArrayList)9 HashSet (java.util.HashSet)9 Set (java.util.Set)9 Stream (java.util.stream.Stream)9 Nonnull (javax.annotation.Nonnull)9 StringUtils (org.apache.commons.lang3.StringUtils)9 Logger (org.slf4j.Logger)9 LoggerFactory (org.slf4j.LoggerFactory)9 DataSet (com.thinkbiganalytics.metadata.api.catalog.DataSet)8 Datasource (com.thinkbiganalytics.metadata.rest.model.data.Datasource)8 RestResponseStatus (com.thinkbiganalytics.rest.model.RestResponseStatus)8