Search in sources :

Example 16 with SqlParameterValue

use of org.springframework.jdbc.core.SqlParameterValue in project metacat by Netflix.

the class MysqlUserMetadataService method _getMetadataMap.

/**
 * get Metadata Map.
 *
 * @param keys list of keys
 * @param sql  query string
 * @return map of the metadata
 */
@SuppressWarnings("checkstyle:methodname")
private Map<String, ObjectNode> _getMetadataMap(@Nullable final List<?> keys, final String sql) {
    final Map<String, ObjectNode> result = Maps.newHashMap();
    if (keys == null || keys.isEmpty()) {
        return result;
    }
    final List<String> paramVariables = keys.stream().map(s -> "?").collect(Collectors.toList());
    final SqlParameterValue[] aKeys = keys.stream().map(o -> new SqlParameterValue(Types.VARCHAR, o.toString())).toArray(SqlParameterValue[]::new);
    final String query = String.format(sql, Joiner.on("," + "").join(paramVariables));
    try {
        final ResultSetExtractor<Void> handler = resultSet -> {
            while (resultSet.next()) {
                final String json = resultSet.getString("data");
                final String name = resultSet.getString("name");
                if (json != null) {
                    try {
                        result.put(name, metacatJson.parseJsonObject(json));
                    } catch (MetacatJsonException e) {
                        log.error("Invalid json '{}' for name '{}'", json, name);
                        throw new UserMetadataServiceException(String.format("Invalid json %s for name %s", json, name), e);
                    }
                }
            }
            return null;
        };
        jdbcTemplate.query(query, aKeys, handler);
    } catch (Exception e) {
        final String message = String.format("Failed to get data for %s", keys);
        log.error(message, e);
        throw new UserMetadataServiceException(message, e);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) InvalidMetadataException(com.netflix.metacat.common.server.connectors.exception.InvalidMetadataException) MetacatJson(com.netflix.metacat.common.json.MetacatJson) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Data(lombok.Data) GetMetadataInterceptorParameters(com.netflix.metacat.common.server.usermetadata.GetMetadataInterceptorParameters) MetadataInterceptor(com.netflix.metacat.common.server.usermetadata.MetadataInterceptor) Optional(java.util.Optional) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) ResultSetExtractor(org.springframework.jdbc.core.ResultSetExtractor) Joiner(com.google.common.base.Joiner) Transactional(org.springframework.transaction.annotation.Transactional) Types(java.sql.Types) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) InvalidMetadataException(com.netflix.metacat.common.server.connectors.exception.InvalidMetadataException) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException)

Example 17 with SqlParameterValue

use of org.springframework.jdbc.core.SqlParameterValue in project metacat by Netflix.

the class DirectSqlGetPartition method getPartitionNames.

/**
 * getPartitionNames.
 *
 * @param context      request context
 * @param uris         uris
 * @param prefixSearch prefixSearch
 * @return partition names
 */
@Transactional(readOnly = true)
public Map<String, List<QualifiedName>> getPartitionNames(@Nonnull final ConnectorRequestContext context, @Nonnull final List<String> uris, final boolean prefixSearch) {
    final long start = registry.clock().wallTime();
    final Map<String, List<QualifiedName>> result = Maps.newHashMap();
    // Create the sql
    final StringBuilder queryBuilder = new StringBuilder(SQL.SQL_GET_PARTITION_NAMES_BY_URI);
    final List<SqlParameterValue> params = Lists.newArrayList();
    if (prefixSearch) {
        queryBuilder.append(" 1=2");
        uris.forEach(uri -> {
            queryBuilder.append(" or location like ?");
            params.add(new SqlParameterValue(Types.VARCHAR, uri + "%"));
        });
    } else {
        queryBuilder.append(" location in (");
        Joiner.on(',').appendTo(queryBuilder, uris.stream().map(uri -> "?").collect(Collectors.toList()));
        queryBuilder.append(")");
        params.addAll(uris.stream().map(uri -> new SqlParameterValue(Types.VARCHAR, uri)).collect(Collectors.toList()));
    }
    final ResultSetExtractor<Map<String, List<QualifiedName>>> handler = rs -> {
        while (rs.next()) {
            final String schemaName = rs.getString("schema_name");
            final String tableName = rs.getString("table_name");
            final String partitionName = rs.getString("partition_name");
            final String uri = rs.getString("location");
            final List<QualifiedName> partitionNames = result.get(uri);
            final QualifiedName qualifiedName = QualifiedName.ofPartition(catalogName, schemaName, tableName, partitionName);
            if (partitionNames == null) {
                result.put(uri, Lists.newArrayList(qualifiedName));
            } else {
                partitionNames.add(qualifiedName);
            }
        }
        return result;
    };
    try {
        jdbcTemplate.query(queryBuilder.toString(), params.toArray(), handler);
    } finally {
        this.fastServiceMetric.recordTimer(HiveMetrics.TagGetPartitionNames.getMetricName(), registry.clock().wallTime() - start);
    }
    return result;
}
Also used : HiveConfigConstants(com.netflix.metacat.connector.hive.util.HiveConfigConstants) HiveConnectorFastServiceMetric(com.netflix.metacat.connector.hive.util.HiveConnectorFastServiceMetric) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) Date(java.util.Date) PartitionFilterGenerator(com.netflix.metacat.connector.hive.util.PartitionFilterGenerator) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) Matcher(java.util.regex.Matcher) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) ConnectorRequestContext(com.netflix.metacat.common.server.connectors.ConnectorRequestContext) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) Joiner(com.google.common.base.Joiner) Sort(com.netflix.metacat.common.dto.Sort) Types(java.sql.Types) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) Functions(com.google.common.base.Functions) Throwables(com.google.common.base.Throwables) Maps(com.google.common.collect.Maps) Table(org.apache.hadoop.hive.metastore.api.Table) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) TimeUnit(java.util.concurrent.TimeUnit) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Futures(com.google.common.util.concurrent.Futures) StringReader(java.io.StringReader) Registry(com.netflix.spectator.api.Registry) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) VisibleForTesting(com.google.common.annotations.VisibleForTesting) ResultSetExtractor(org.springframework.jdbc.core.ResultSetExtractor) Transactional(org.springframework.transaction.annotation.Transactional) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) QualifiedName(com.netflix.metacat.common.QualifiedName) List(java.util.List) Map(java.util.Map) Transactional(org.springframework.transaction.annotation.Transactional)

Example 18 with SqlParameterValue

use of org.springframework.jdbc.core.SqlParameterValue in project metacat by Netflix.

the class DirectSqlSavePartition method getPartitionSequenceIds.

private List<PartitionSequenceIds> getPartitionSequenceIds(final QualifiedName tableName, final List<String> partitionNames) {
    final List<String> paramVariables = partitionNames.stream().map(s -> "?").collect(Collectors.toList());
    final String paramVariableString = Joiner.on(",").skipNulls().join(paramVariables);
    final SqlParameterValue[] values = new SqlParameterValue[partitionNames.size() + 2];
    int index = 0;
    values[index++] = new SqlParameterValue(Types.VARCHAR, tableName.getDatabaseName());
    values[index++] = new SqlParameterValue(Types.VARCHAR, tableName.getTableName());
    for (String partitionName : partitionNames) {
        values[index++] = new SqlParameterValue(Types.VARCHAR, partitionName);
    }
    return jdbcTemplate.query(String.format(SQL.PARTITIONS_SELECT, paramVariableString), values, (rs, rowNum) -> new PartitionSequenceIds(rs.getLong("part_id"), rs.getLong("sd_id"), rs.getLong("serde_id")));
}
Also used : HiveConnectorFastServiceMetric(com.netflix.metacat.connector.hive.util.HiveConnectorFastServiceMetric) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) Lists(com.google.common.collect.Lists) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) EmptyResultDataAccessException(org.springframework.dao.EmptyResultDataAccessException) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) Table(org.apache.hadoop.hive.metastore.api.Table) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) DuplicateKeyException(org.springframework.dao.DuplicateKeyException) List(java.util.List) PartitionAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.PartitionAlreadyExistsException) PartitionUtil(com.netflix.metacat.connector.hive.util.PartitionUtil) Registry(com.netflix.spectator.api.Registry) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Joiner(com.google.common.base.Joiner) Transactional(org.springframework.transaction.annotation.Transactional) Types(java.sql.Types) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue)

Example 19 with SqlParameterValue

use of org.springframework.jdbc.core.SqlParameterValue in project metacat by Netflix.

the class DirectSqlTable method delete.

/**
 * Deletes all the table related information from the store.
 * @param tableName table name
 */
public void delete(final QualifiedName tableName) {
    try {
        final TableSequenceIds ids = getSequenceIds(tableName);
        directSqlSavePartition.delete(tableName);
        jdbcTemplate.update(SQL.UPDATE_SDS_CD, new SqlParameterValue(Types.BIGINT, null), new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.UPDATE_SDS_SERDE, new SqlParameterValue(Types.BIGINT, null), new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        // 
        try {
            jdbcTemplate.update(SQL.DELETE_COLUMNS_OLD, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        } catch (DataAccessException ignored) {
            log.debug("Ignore. Probably table COLUMNS_OLD does not exist.");
        }
        try {
            jdbcTemplate.update(SQL.DELETE_TBL_PRIVS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        } catch (DataAccessException ignored) {
            log.debug("Ignore. Probably table TBL_PRIVS does not exist.");
        }
        try {
            jdbcTemplate.update(SQL.DELETE_TBL_COL_PRIVS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        } catch (DataAccessException ignored) {
            log.debug("Ignore. Probably table TBL_COL_PRIVS does not exist.");
        }
        jdbcTemplate.update(SQL.DELETE_COLUMNS_V2, new SqlParameterValue(Types.BIGINT, ids.getCdId()));
        jdbcTemplate.update(SQL.DELETE_CDS, new SqlParameterValue(Types.BIGINT, ids.getCdId()));
        jdbcTemplate.update(SQL.DELETE_PARTITION_KEYS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        jdbcTemplate.update(SQL.DELETE_TABLE_PARAMS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        jdbcTemplate.update(SQL.DELETE_TAB_COL_STATS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        jdbcTemplate.update(SQL.UPDATE_TABLE_SD, new SqlParameterValue(Types.BIGINT, null), new SqlParameterValue(Types.BIGINT, ids.getTableId()));
        jdbcTemplate.update(SQL.DELETE_SKEWED_COL_NAMES, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_BUCKETING_COLS, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_SORT_COLS, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_SD_PARAMS, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_SKEWED_COL_VALUE_LOC_MAP, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_SKEWED_VALUES, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_SERDE_PARAMS, new SqlParameterValue(Types.BIGINT, ids.getSerdeId()));
        jdbcTemplate.update(SQL.DELETE_SERDES, new SqlParameterValue(Types.BIGINT, ids.getSerdeId()));
        jdbcTemplate.update(SQL.DELETE_SDS, new SqlParameterValue(Types.BIGINT, ids.getSdsId()));
        jdbcTemplate.update(SQL.DELETE_TBLS, new SqlParameterValue(Types.BIGINT, ids.getTableId()));
    } catch (DataAccessException e) {
        throw new ConnectorException(String.format("Failed delete hive table %s", tableName), e);
    }
}
Also used : SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) DataAccessException(org.springframework.dao.DataAccessException) EmptyResultDataAccessException(org.springframework.dao.EmptyResultDataAccessException)

Aggregations

SqlParameterValue (org.springframework.jdbc.core.SqlParameterValue)19 Test (org.junit.Test)8 SQLException (java.sql.SQLException)7 Map (java.util.Map)6 Lists (com.google.common.collect.Lists)5 QualifiedName (com.netflix.metacat.common.QualifiedName)5 Types (java.sql.Types)5 List (java.util.List)5 Set (java.util.Set)5 Slf4j (lombok.extern.slf4j.Slf4j)5 EmptyResultDataAccessException (org.springframework.dao.EmptyResultDataAccessException)5 JdbcTemplate (org.springframework.jdbc.core.JdbcTemplate)5 Transactional (org.springframework.transaction.annotation.Transactional)5 Joiner (com.google.common.base.Joiner)4 Maps (com.google.common.collect.Maps)4 ConnectorException (com.netflix.metacat.common.server.connectors.exception.ConnectorException)4 UserMetadataServiceException (com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException)4 Date (java.util.Date)4 Collectors (java.util.stream.Collectors)4 ResultSetExtractor (org.springframework.jdbc.core.ResultSetExtractor)4