Search in sources :

Example 1 with DefaultJdbcTable

use of com.thinkbiganalytics.schema.DefaultJdbcTable in project kylo by Teradata.

the class HiveMetastoreService method listTables.

@Nonnull
public List<JdbcTable> listTables(@Nullable final String catalog, @Nullable final String schema, @Nullable final String pattern, @Nullable final Pageable pageable) {
    String query = "SELECT d.NAME as \"DATABASE_NAME\", t.TBL_NAME, t.TBL_TYPE, p.PARAM_VALUE" + " FROM TBLS t" + " JOIN DBS d on d.DB_ID = t.DB_ID" + " LEFT JOIN TABLE_PARAMS p on p.TBL_ID = t.TBL_ID AND p.PARAM_KEY = \"comment\"" + " WHERE d.NAME LIKE ? " + " AND t.TBL_NAME LIKE ? " + " ORDER BY \"DATABASE_NAME\", t.TBL_NAME";
    if (DatabaseType.POSTGRES.equals(getMetastoreDatabaseType())) {
        query = "SELECT d.\"NAME\" as \"DATABASE_NAME\", t.\"TBL_NAME\", t.\"TBL_TYPE\", p.\"PARAM_VALUE\"" + " FROM \"TBLS\" t" + " JOIN \"DBS\" d on d.\"DB_ID\" = t.\"DB_ID\" " + " LEFT JOIN \"TABLE_PARAMS\" p ON p.\"TBL_ID\" = t.\"TBL_ID\" and p.\"PARAM_KEY\" = 'comment'" + " WHERE d.\"NAME\" LIKE ? " + " AND t.\"TBL_NAME\" LIKE ? " + " ORDER BY d.\"NAME\", t.\"TBL_NAME\"";
    }
    final List<JdbcTable> tables = hiveMetatoreJdbcTemplate.query(query, ps -> {
        ps.setString(1, schema == null ? "%" : schema);
        ps.setString(2, pattern == null ? "%" : "%" + pattern + "%");
    }, (rs, i) -> {
        final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(rs.getString("TBL_NAME"), rs.getString("TBL_TYPE"));
        jdbcTable.setIdentifierQuoteString("`");
        jdbcTable.setRemarks(rs.getString("PARAM_VALUE"));
        jdbcTable.setSchema(rs.getString("DATABASE_NAME"));
        return jdbcTable;
    });
    if (userImpersonationEnabled) {
        return tables.stream().filter(jdbcTable -> hiveService.isTableAccessibleByImpersonatedUser(jdbcTable.getSchema() + "." + jdbcTable.getName())).collect(Collectors.toList());
    } else {
        return tables;
    }
}
Also used : DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) DataAccessException(org.springframework.dao.DataAccessException) DatabaseMetadata(com.thinkbiganalytics.discovery.schema.DatabaseMetadata) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) HashMap(java.util.HashMap) MetaDataAccessException(org.springframework.jdbc.support.MetaDataAccessException) DefaultField(com.thinkbiganalytics.discovery.model.DefaultField) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) SQLException(java.sql.SQLException) DefaultDatabaseMetadata(com.thinkbiganalytics.discovery.model.DefaultDatabaseMetadata) Service(org.springframework.stereotype.Service) ResultSet(java.sql.ResultSet) Map(java.util.Map) Qualifier(org.springframework.beans.factory.annotation.Qualifier) DataSource(javax.sql.DataSource) Pageable(org.springframework.data.domain.Pageable) DefaultJdbcTable(com.thinkbiganalytics.schema.DefaultJdbcTable) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) Logger(org.slf4j.Logger) DefaultJdbcSchema(com.thinkbiganalytics.schema.DefaultJdbcSchema) JdbcCatalog(com.thinkbiganalytics.discovery.schema.JdbcCatalog) Collectors(java.util.stream.Collectors) List(java.util.List) Field(com.thinkbiganalytics.discovery.schema.Field) JdbcSchema(com.thinkbiganalytics.discovery.schema.JdbcSchema) RowMapper(org.springframework.jdbc.core.RowMapper) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DefaultTableSchema(com.thinkbiganalytics.discovery.model.DefaultTableSchema) JdbcTable(com.thinkbiganalytics.discovery.schema.JdbcTable) Collections(java.util.Collections) DefaultJdbcTable(com.thinkbiganalytics.schema.DefaultJdbcTable) DefaultJdbcTable(com.thinkbiganalytics.schema.DefaultJdbcTable) JdbcTable(com.thinkbiganalytics.discovery.schema.JdbcTable) Nonnull(javax.annotation.Nonnull)

Example 2 with DefaultJdbcTable

use of com.thinkbiganalytics.schema.DefaultJdbcTable in project kylo by Teradata.

the class DefaultCatalogTableManager method describeTable.

@Nonnull
@Override
public CatalogTableSchema describeTable(@Nonnull final DataSource dataSource, @Nullable final String schemaName, @Nullable final String tableName) throws SQLException {
    final DataSetTemplate template = DataSourceUtil.mergeTemplates(dataSource);
    if (Objects.equals("hive", template.getFormat())) {
        final TableSchema tableSchema = hiveMetastoreService.getTable(schemaName, tableName);
        final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
        // Get table metadata
        if (StringUtils.isNotEmpty(tableSchema.getName())) {
            final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setCatalog(tableSchema.getDatabaseName());
            jdbcTable.setRemarks(tableSchema.getDescription());
            jdbcTable.setSchema(tableSchema.getSchemaName());
            jdbcTable.setCatalogSeparator(".");
            jdbcTable.setIdentifierQuoteString("`");
            catalogTableSchema.setTable(createTable(jdbcTable));
        }
        return catalogTableSchema;
    } else if (Objects.equals("jdbc", template.getFormat())) {
        return isolatedFunction(template, schemaName, (connection, schemaParser) -> {
            final javax.sql.DataSource ds = new SingleConnectionDataSource(connection, true);
            final DBSchemaParser tableSchemaParser = new DBSchemaParser(ds, new KerberosTicketConfiguration());
            final TableSchema tableSchema = tableSchemaParser.describeTable(schemaName, tableName);
            if (tableSchema != null) {
                // Get table metadata
                final DefaultJdbcTable jdbcTable = new DefaultJdbcTable(tableSchema.getName(), "TABLE");
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setCatalog(tableSchema.getDatabaseName());
                jdbcTable.setRemarks(tableSchema.getDescription());
                jdbcTable.setSchema(tableSchema.getSchemaName());
                jdbcTable.setMetaData(connection.getMetaData());
                // Return table schema
                final CatalogTableSchema catalogTableSchema = new CatalogTableSchema(tableSchema);
                catalogTableSchema.setTable(createTable(jdbcTable));
                return catalogTableSchema;
            } else {
                return null;
            }
        });
    } else {
        throw new IllegalArgumentException("Unsupported format: " + template.getFormat());
    }
}
Also used : PoolingDataSourceService(com.thinkbiganalytics.db.PoolingDataSourceService) DatabaseType(com.thinkbiganalytics.jdbc.util.DatabaseType) LoadingCache(com.google.common.cache.LoadingCache) Connection(java.sql.Connection) DataSourceUtil(com.thinkbiganalytics.kylo.catalog.datasource.DataSourceUtil) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) DataSourceProperties(com.thinkbiganalytics.db.DataSourceProperties) DataSetUtil(com.thinkbiganalytics.kylo.catalog.dataset.DataSetUtil) StringUtils(org.apache.commons.lang3.StringUtils) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) Value(org.springframework.beans.factory.annotation.Value) SingleConnectionDataSource(org.springframework.jdbc.datasource.SingleConnectionDataSource) HashSet(java.util.HashSet) SQLException(java.sql.SQLException) DBSchemaParser(com.thinkbiganalytics.schema.DBSchemaParser) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Configuration(org.apache.hadoop.conf.Configuration) Qualifier(org.springframework.beans.factory.annotation.Qualifier) CatalogTableSchema(com.thinkbiganalytics.discovery.model.CatalogTableSchema) HiveMetastoreService(com.thinkbiganalytics.hive.service.HiveMetastoreService) DefaultJdbcTable(com.thinkbiganalytics.schema.DefaultJdbcTable) Nonnull(javax.annotation.Nonnull) JdbcSchemaParser(com.thinkbiganalytics.discovery.schema.JdbcSchemaParser) Nullable(javax.annotation.Nullable) Logger(org.slf4j.Logger) Properties(java.util.Properties) Set(java.util.Set) JdbcCatalog(com.thinkbiganalytics.discovery.schema.JdbcCatalog) HadoopClassLoader(com.thinkbiganalytics.kylo.util.HadoopClassLoader) Collectors(java.util.stream.Collectors) JdbcSchemaParserProvider(com.thinkbiganalytics.schema.JdbcSchemaParserProvider) KerberosUtil(com.thinkbiganalytics.kerberos.KerberosUtil) CacheLoader(com.google.common.cache.CacheLoader) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) Component(org.springframework.stereotype.Component) List(java.util.List) JdbcSchema(com.thinkbiganalytics.discovery.schema.JdbcSchema) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) KerberosTicketConfiguration(com.thinkbiganalytics.kerberos.KerberosTicketConfiguration) VisibleForTesting(com.google.common.annotations.VisibleForTesting) CacheBuilder(com.google.common.cache.CacheBuilder) JdbcTable(com.thinkbiganalytics.discovery.schema.JdbcTable) DataSetTable(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTable) SingleConnectionDataSource(org.springframework.jdbc.datasource.SingleConnectionDataSource) DBSchemaParser(com.thinkbiganalytics.schema.DBSchemaParser) CatalogTableSchema(com.thinkbiganalytics.discovery.model.CatalogTableSchema) TableSchema(com.thinkbiganalytics.discovery.schema.TableSchema) DataSetTemplate(com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate) DefaultJdbcTable(com.thinkbiganalytics.schema.DefaultJdbcTable) KerberosTicketConfiguration(com.thinkbiganalytics.kerberos.KerberosTicketConfiguration) CatalogTableSchema(com.thinkbiganalytics.discovery.model.CatalogTableSchema) SingleConnectionDataSource(org.springframework.jdbc.datasource.SingleConnectionDataSource) DataSource(com.thinkbiganalytics.kylo.catalog.rest.model.DataSource) Nonnull(javax.annotation.Nonnull)

Aggregations

JdbcCatalog (com.thinkbiganalytics.discovery.schema.JdbcCatalog)2 JdbcSchema (com.thinkbiganalytics.discovery.schema.JdbcSchema)2 JdbcTable (com.thinkbiganalytics.discovery.schema.JdbcTable)2 TableSchema (com.thinkbiganalytics.discovery.schema.TableSchema)2 DatabaseType (com.thinkbiganalytics.jdbc.util.DatabaseType)2 DefaultJdbcTable (com.thinkbiganalytics.schema.DefaultJdbcTable)2 SQLException (java.sql.SQLException)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2 Nonnull (javax.annotation.Nonnull)2 Nullable (javax.annotation.Nullable)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 Autowired (org.springframework.beans.factory.annotation.Autowired)2 Qualifier (org.springframework.beans.factory.annotation.Qualifier)2 Value (org.springframework.beans.factory.annotation.Value)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 CacheBuilder (com.google.common.cache.CacheBuilder)1 CacheLoader (com.google.common.cache.CacheLoader)1 LoadingCache (com.google.common.cache.LoadingCache)1