Search in sources :

Example 6 with ResultSetHandler

use of org.apache.commons.dbutils.ResultSetHandler in project metacat by Netflix.

the class HiveConnectorFastPartitionService method getPartitionKeys.

private List<FieldSchema> getPartitionKeys(final String databaseName, final String tableName) {
    // Get data source
    final DataSource dataSource = DataSourceManager.get().get(catalogName);
    final ResultSetHandler<List<FieldSchema>> handler = rs -> {
        final List<FieldSchema> result = Lists.newArrayList();
        while (rs.next()) {
            final String name = rs.getString("pkey_name");
            final String type = rs.getString("pkey_type");
            result.add(new FieldSchema(name, type, null));
        }
        return result;
    };
    try (Connection conn = dataSource.getConnection()) {
        return new QueryRunner().query(conn, SQL_GET_PARTITION_KEYS, handler, databaseName, tableName);
    } catch (SQLException e) {
        throw Throwables.propagate(e);
    }
}
Also used : Connection(java.sql.Connection) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) Date(java.util.Date) PartitionFilterGenerator(com.netflix.metacat.connector.hive.util.PartitionFilterGenerator) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) QueryRunner(org.apache.commons.dbutils.QueryRunner) NonNull(lombok.NonNull) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) Joiner(com.google.common.base.Joiner) Sort(com.netflix.metacat.common.dto.Sort) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) HashMap(java.util.HashMap) Id(com.netflix.spectator.api.Id) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) DataSource(javax.sql.DataSource) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) PartitionDetail(com.netflix.metacat.connector.hive.util.PartitionDetail) Functions(com.google.common.base.Functions) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Throwables(com.google.common.base.Throwables) Maps(com.google.common.collect.Maps) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) TimeUnit(java.util.concurrent.TimeUnit) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Futures(com.google.common.util.concurrent.Futures) StringReader(java.io.StringReader) Registry(com.netflix.spectator.api.Registry) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) SQLException(java.sql.SQLException) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Connection(java.sql.Connection) List(java.util.List) ArrayList(java.util.ArrayList) QueryRunner(org.apache.commons.dbutils.QueryRunner) DataSource(javax.sql.DataSource)

Example 7 with ResultSetHandler

use of org.apache.commons.dbutils.ResultSetHandler in project metacat by Netflix.

the class HiveConnectorFastTableService method getTableNames.

@Override
public Map<String, List<QualifiedName>> getTableNames(@Nonnull final ConnectorContext context, @Nonnull final List<String> uris, final boolean prefixSearch) {
    final long start = registry.clock().monotonicTime();
    final Map<String, String> tags = new HashMap<String, String>();
    tags.put("request", HiveMetrics.getTableNames.name());
    final Map<String, List<QualifiedName>> result = Maps.newHashMap();
    // Get data source
    final DataSource dataSource = DataSourceManager.get().get(catalogName);
    // Create the sql
    final StringBuilder queryBuilder = new StringBuilder(SQL_GET_TABLE_NAMES_BY_URI);
    final List<String> params = Lists.newArrayList();
    if (prefixSearch) {
        queryBuilder.append(" and (1=0");
        uris.forEach(uri -> {
            queryBuilder.append(" or location like ?");
            params.add(uri + "%");
        });
        queryBuilder.append(" )");
    } else {
        queryBuilder.append(" and location in (");
        uris.forEach(uri -> {
            queryBuilder.append("?,");
            params.add(uri);
        });
        queryBuilder.deleteCharAt(queryBuilder.length() - 1).append(")");
    }
    // Handler for reading the result set
    ResultSetHandler<Map<String, List<QualifiedName>>> handler = rs -> {
        while (rs.next()) {
            final String schemaName = rs.getString("schema_name");
            final String tableName = rs.getString("table_name");
            final String uri = rs.getString("location");
            List<QualifiedName> names = result.get(uri);
            if (names == null) {
                names = Lists.newArrayList();
                result.put(uri, names);
            }
            names.add(QualifiedName.ofTable(catalogName, schemaName, tableName));
        }
        return result;
    };
    try (Connection conn = dataSource.getConnection()) {
        new QueryRunner().query(conn, queryBuilder.toString(), handler, params.toArray());
    } catch (SQLException e) {
        throw Throwables.propagate(e);
    } finally {
        final long duration = registry.clock().monotonicTime() - start;
        log.debug("### Time taken to complete getTableNames is {} ms", duration);
        this.registry.timer(requestTimerId.withTags(tags)).record(duration, TimeUnit.MILLISECONDS);
    }
    return result;
}
Also used : Connection(java.sql.Connection) HashMap(java.util.HashMap) Id(com.netflix.spectator.api.Id) Inject(javax.inject.Inject) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) ScalarHandler(org.apache.commons.dbutils.handlers.ScalarHandler) Map(java.util.Map) DataSource(javax.sql.DataSource) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) Nonnull(javax.annotation.Nonnull) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) QueryRunner(org.apache.commons.dbutils.QueryRunner) NonNull(lombok.NonNull) Throwables(com.google.common.base.Throwables) QualifiedName(com.netflix.metacat.common.QualifiedName) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Registry(com.netflix.spectator.api.Registry) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) HashMap(java.util.HashMap) SQLException(java.sql.SQLException) QualifiedName(com.netflix.metacat.common.QualifiedName) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DataSource(javax.sql.DataSource) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map)

Example 8 with ResultSetHandler

use of org.apache.commons.dbutils.ResultSetHandler in project metacat by Netflix.

the class MysqlUserMetadataService method _getMetadataMap.

@SuppressWarnings("checkstyle:methodname")
private Map<String, ObjectNode> _getMetadataMap(@Nullable final List<?> keys, final String sql) {
    final Map<String, ObjectNode> result = Maps.newHashMap();
    if (keys == null || keys.isEmpty()) {
        return result;
    }
    final List<String> paramVariables = keys.stream().map(s -> "?").collect(Collectors.toList());
    final String[] aKeys = keys.stream().map(Object::toString).toArray(String[]::new);
    final String query = String.format(sql, Joiner.on(",").join(paramVariables));
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        final ResultSetHandler<Void> handler = resultSet -> {
            while (resultSet.next()) {
                final String json = resultSet.getString("data");
                final String name = resultSet.getString("name");
                if (json != null) {
                    try {
                        result.put(name, metacatJson.parseJsonObject(json));
                    } catch (MetacatJsonException e) {
                        log.error("Invalid json '{}' for name '{}'", json, name);
                        throw new UserMetadataServiceException(String.format("Invalid json %s for name %s", json, name), e);
                    }
                }
            }
            return null;
        };
        new QueryRunner().query(connection, query, handler, (Object[]) aKeys);
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException(String.format("Failed to get data for %s", keys), e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) SQLException(java.sql.SQLException) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner)

Example 9 with ResultSetHandler

use of org.apache.commons.dbutils.ResultSetHandler in project metacat by Netflix.

the class MysqlUserMetadataService method searchByOwners.

@Override
public List<QualifiedName> searchByOwners(final Set<String> owners) {
    final List<QualifiedName> result = Lists.newArrayList();
    final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATA_NAMES);
    final List<Object> paramList = Lists.newArrayList();
    query.append(" where 1=0");
    owners.forEach(s -> {
        query.append(" or data like ?");
        paramList.add("%\"userId\":\"" + s.trim() + "\"%");
    });
    final Object[] params = new Object[paramList.size()];
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        // Handler for reading the result set
        final ResultSetHandler<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                result.add(QualifiedName.fromString(definitionName, false));
            }
            return null;
        };
        new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to get definition data", e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) SQLException(java.sql.SQLException) QualifiedName(com.netflix.metacat.common.QualifiedName) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner)

Example 10 with ResultSetHandler

use of org.apache.commons.dbutils.ResultSetHandler in project metacat by Netflix.

the class MysqlUserMetadataService method searchDefinitionMetadatas.

@Override
public List<DefinitionMetadataDto> searchDefinitionMetadatas(final Set<String> propertyNames, final String type, final String name, final String sortBy, final String sortOrder, final Integer offset, final Integer limit) {
    final List<DefinitionMetadataDto> result = Lists.newArrayList();
    final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATAS);
    final List<Object> paramList = Lists.newArrayList();
    if (type != null) {
        String typeRegex = null;
        switch(type) {
            case "database":
                typeRegex = "^[^/]*/[^/]*$";
                break;
            case "table":
                typeRegex = "^[^/]*/[^/]*/[^/]*$";
                break;
            case "partition":
                typeRegex = "^[^/]*/[^/]*/[^/]*/.*$";
                break;
            default:
        }
        if (typeRegex != null) {
            query.append(" and name rlike ?");
            paramList.add(typeRegex);
        }
    }
    if (propertyNames != null && !propertyNames.isEmpty()) {
        propertyNames.forEach(propertyName -> {
            query.append(" and data like ?");
            paramList.add("%\"" + propertyName + "\":%");
        });
    }
    if (!Strings.isNullOrEmpty(name)) {
        query.append(" and name like ?");
        paramList.add(name);
    }
    if (!Strings.isNullOrEmpty(sortBy)) {
        query.append(" order by ").append(sortBy);
        if (!Strings.isNullOrEmpty(sortOrder)) {
            query.append(" ").append(sortOrder);
        }
    }
    if (limit != null) {
        query.append(" limit ");
        if (offset != null) {
            query.append(offset).append(",");
        }
        query.append(limit);
    }
    final Object[] params = new Object[paramList.size()];
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        // Handler for reading the result set
        final ResultSetHandler<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                final String data = rs.getString("data");
                final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
                definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
                definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
                result.add(definitionMetadataDto);
            }
            return null;
        };
        new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to get definition data", e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) SQLException(java.sql.SQLException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto)

Aggregations

Lists (com.google.common.collect.Lists)10 Maps (com.google.common.collect.Maps)10 QualifiedName (com.netflix.metacat.common.QualifiedName)10 DataSourceManager (com.netflix.metacat.common.server.util.DataSourceManager)10 Connection (java.sql.Connection)10 SQLException (java.sql.SQLException)10 List (java.util.List)10 Map (java.util.Map)10 Joiner (com.google.common.base.Joiner)9 Strings (com.google.common.base.Strings)9 Date (java.util.Date)9 Nonnull (javax.annotation.Nonnull)8 DataSource (javax.sql.DataSource)8 Slf4j (lombok.extern.slf4j.Slf4j)8 QueryRunner (org.apache.commons.dbutils.QueryRunner)8 ResultSetHandler (org.apache.commons.dbutils.ResultSetHandler)8 Throwables (com.google.common.base.Throwables)7 ConnectorContext (com.netflix.metacat.common.server.connectors.ConnectorContext)7 ThreadServiceManager (com.netflix.metacat.common.server.util.ThreadServiceManager)7 HiveConnectorInfoConverter (com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter)7