Examples with Datasource com.thinkbiganalytics.metadata.rest.model.data.Datasource used on opensource projects

Example 11 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DatasourceController method getTablesAndColumns.

/**
 * Gets the tables and their columns from the specified data source for given schema
 *
 * @param idStr  the data source id
 * @param schema the schema name, or {@code null} for all schemas
 * @return the list of tables and their columns
 */
@GET
@Path("{id}/table-columns")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Gets the tables and their columns from the data source for given schema", notes = "Connects to the database specified by the data source.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns tables and columns", response = String.class, responseContainer = "List"), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 404, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response getTablesAndColumns(@PathParam("id") final String idStr, @QueryParam("schema") final String schema) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasetProvider.getDatasource(datasetProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Retrieve table names using system user
    return metadata.read(() -> {
        JdbcDatasource datasource = id.map(datasetProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        List<String> tableNamesForDatasource = dbcpConnectionPoolTableInfo.getTableNamesForDatasource(datasource, schema, null);
        List<DatabaseMetadata> tables = new ArrayList<>();
        if (tableNamesForDatasource != null) {
            tables = tableNamesForDatasource.stream().flatMap(schemaNameDotTableName -> {
                String tableName = schemaNameDotTableName.substring(schema.length() + 1);
                TableSchema tableSchema = dbcpConnectionPoolTableInfo.describeTableForDatasource(datasource, schema, tableName);
                return tableSchema.getFields().stream().map(field -> {
                    DefaultDatabaseMetadata meta = new DefaultDatabaseMetadata();
                    meta.setDatabaseName(schema);
                    meta.setColumnName(field.getName());
                    meta.setTableName(tableName);
                    return meta;
                });
            }).collect(Collectors.toList());
        }
        return Response.ok(tables).build();
    }, MetadataAccess.SERVICE);
}

Example 12 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class DatasourceController method previewQuery.

/**
 * Executes a query on the specified datasource.
 *
 * @param idStr the datasource id
 * @param query the SQL query
 * @return the SQL result
 */
@POST
@Path("{id}/preview/{schema}/{table}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Executes a preview query appropriate for the type of datasource and returns the result.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the result.", response = QueryResult.class), @ApiResponse(code = 403, message = "Access denied.", response = RestResponseStatus.class), @ApiResponse(code = 400, message = "A JDBC data source with that id does not exist.", response = RestResponseStatus.class), @ApiResponse(code = 500, message = "NiFi or the database are unavailable.", response = RestResponseStatus.class) })
public Response previewQuery(@PathParam("id") final String idStr, @PathParam("schema") final String schema, @PathParam("table") final String tableName, @QueryParam("limit") @DefaultValue("10") final int limit) {
    // Verify user has access to data source
    final Optional<com.thinkbiganalytics.metadata.api.datasource.Datasource.ID> id = metadata.read(() -> {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ACCESS_DATASOURCES);
        final com.thinkbiganalytics.metadata.api.datasource.Datasource datasource = datasetProvider.getDatasource(datasetProvider.resolve(idStr));
        return Optional.ofNullable(datasource).map(com.thinkbiganalytics.metadata.api.datasource.Datasource::getId);
    });
    // Execute query
    return metadata.read(() -> {
        final QueryResult result = id.map(datasetProvider::getDatasource).map(ds -> datasourceTransform.toDatasource(ds, DatasourceModelTransform.Level.ADMIN)).filter(JdbcDatasource.class::isInstance).map(JdbcDatasource.class::cast).map(datasource -> dbcpConnectionPoolTableInfo.executePreviewQueryForDatasource(datasource, schema, tableName, limit)).orElseThrow(() -> new NotFoundException("No JDBC datasource exists with the given ID: " + idStr));
        return Response.ok(result).build();
    }, MetadataAccess.SERVICE);
}

Example 13 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class FeedLineage method serialize.

@JsonIgnore
private void serialize(Feed feed) {
    if (feed.getDependentFeeds() != null) {
        Set<String> ids = new HashSet<>();
        Set<Feed> dependentFeeds = new HashSet<>(feed.getDependentFeeds());
        feed.setDependentFeeds(null);
        dependentFeeds.stream().forEach(depFeed -> {
            feedMap.put(depFeed.getId(), depFeed);
            ids.add(depFeed.getId());
            if (depFeed.getDependentFeeds() != null) {
                serialize(depFeed);
            }
        });
        feed.setDependentFeedIds(ids);
    }
    if (feed.getUsedByFeeds() != null) {
        Set<String> ids = new HashSet<>();
        Set<Feed> usedByFeeds = new HashSet<>(feed.getUsedByFeeds());
        feed.getUsedByFeeds().clear();
        usedByFeeds.stream().forEach(depFeed -> {
            feedMap.put(depFeed.getId(), depFeed);
            ids.add(depFeed.getId());
            if (depFeed.getUsedByFeeds() != null) {
                serialize(depFeed);
            }
        });
        feed.setUsedByFeedIds(ids);
    }
    if (feed.getSources() != null) {
        feed.getSources().forEach(feedSource -> {
            Datasource ds = serializeDatasource(feedSource.getDatasource());
            feedSource.setDatasource(null);
            if (StringUtils.isBlank(feedSource.getDatasourceId())) {
                feedSource.setDatasourceId(ds != null ? ds.getId() : null);
            }
        });
    }
    if (feed.getDestinations() != null) {
        feed.getDestinations().forEach(feedDestination -> {
            Datasource ds = serializeDatasource(feedDestination.getDatasource());
            feedDestination.setDatasource(null);
            if (StringUtils.isBlank(feedDestination.getDatasourceId())) {
                feedDestination.setDatasourceId(ds != null ? ds.getId() : null);
            }
        });
    }
    feedMap.put(feed.getId(), feed);
}

Example 14 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testGetDatasourceByName.

@Test
public void testGetDatasourceByName() {
    this.provider.ensureDirectoryDatasource("test2", "", Paths.get("aaa", "bbb"));
    Datasource ds = this.provider.getDatasourceByName("test2");
    assertThat(ds).isNotNull();
}

Example 15 with Datasource

use of com.thinkbiganalytics.metadata.rest.model.data.Datasource in project kylo by Teradata.

the class MetadataClientProviderTest method testEnsureFeedSource.

@Test
public void testEnsureFeedSource() {
    Feed feed = this.provider.ensureFeed("category", "test3", "");
    Datasource ds = this.provider.ensureDirectoryDatasource("test3", "", Paths.get("aaa", "bbb"));
    feed = this.provider.ensureFeedSource(feed.getId(), ds.getId());
    assertThat(feed.getSources()).hasSize(1);
    feed = this.provider.ensureFeedSource(feed.getId(), ds.getId());
    assertThat(feed.getSources()).hasSize(1);
}