Search in sources :

Example 6 with QueryRunner

use of org.apache.commons.dbutils.QueryRunner in project metacat by Netflix.

the class HiveConnectorFastPartitionService method getPartitionCount.

/**
     * Number of partitions for the given table.
     *
     * @param tableName tableName
     * @return Number of partitions
     */
@Override
public int getPartitionCount(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName) {
    final long start = registry.clock().monotonicTime();
    final Map<String, String> tags = new HashMap<String, String>();
    tags.put("request", HiveMetrics.getPartitionCount.name());
    final Integer result;
    final DataSource dataSource = DataSourceManager.get().get(catalogName);
    try (Connection conn = dataSource.getConnection()) {
        // Handler for reading the result set
        final ResultSetHandler<Integer> handler = rs -> {
            int count = 0;
            while (rs.next()) {
                count = rs.getInt("count");
            }
            return count;
        };
        result = new QueryRunner().query(conn, SQL_GET_PARTITION_COUNT, handler, tableName.getDatabaseName(), tableName.getTableName());
    } catch (SQLException e) {
        throw new ConnectorException("getPartitionCount", e);
    } finally {
        final long duration = registry.clock().monotonicTime() - start;
        log.debug("### Time taken to complete getPartitionCount is {} ms", duration);
        this.registry.timer(requestTimerId.withTags(tags)).record(duration, TimeUnit.MILLISECONDS);
    }
    return result;
}
Also used : Connection(java.sql.Connection) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) Date(java.util.Date) PartitionFilterGenerator(com.netflix.metacat.connector.hive.util.PartitionFilterGenerator) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) QueryRunner(org.apache.commons.dbutils.QueryRunner) NonNull(lombok.NonNull) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) Joiner(com.google.common.base.Joiner) Sort(com.netflix.metacat.common.dto.Sort) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) HashMap(java.util.HashMap) Id(com.netflix.spectator.api.Id) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) DataSource(javax.sql.DataSource) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) PartitionDetail(com.netflix.metacat.connector.hive.util.PartitionDetail) Functions(com.google.common.base.Functions) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Throwables(com.google.common.base.Throwables) Maps(com.google.common.collect.Maps) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) TimeUnit(java.util.concurrent.TimeUnit) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Futures(com.google.common.util.concurrent.Futures) StringReader(java.io.StringReader) Registry(com.netflix.spectator.api.Registry) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) HashMap(java.util.HashMap) SQLException(java.sql.SQLException) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DataSource(javax.sql.DataSource)

Example 7 with QueryRunner

use of org.apache.commons.dbutils.QueryRunner in project metacat by Netflix.

the class HiveConnectorFastTableService method exists.

/**
     * {@inheritDoc}.
     */
@Override
public boolean exists(@Nonnull final ConnectorContext requestContext, @Nonnull final QualifiedName name) {
    final long start = registry.clock().monotonicTime();
    final Map<String, String> tags = new HashMap<String, String>();
    tags.put("request", HiveMetrics.exists.name());
    boolean result = false;
    // Get data source
    final DataSource dataSource = DataSourceManager.get().get(catalogName);
    try (Connection conn = dataSource.getConnection()) {
        final Object qResult = new QueryRunner().query(conn, SQL_EXIST_TABLE_BY_NAME, new ScalarHandler(1), name.getDatabaseName(), name.getTableName());
        if (qResult != null) {
            result = true;
        }
    } catch (SQLException e) {
        throw Throwables.propagate(e);
    } finally {
        final long duration = registry.clock().monotonicTime() - start;
        log.debug("### Time taken to complete exists is {} ms", duration);
        this.registry.timer(requestTimerId.withTags(tags)).record(duration, TimeUnit.MILLISECONDS);
    }
    return result;
}
Also used : HashMap(java.util.HashMap) SQLException(java.sql.SQLException) ScalarHandler(org.apache.commons.dbutils.handlers.ScalarHandler) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DataSource(javax.sql.DataSource)

Example 8 with QueryRunner

use of org.apache.commons.dbutils.QueryRunner in project metacat by Netflix.

the class MySqlTagService method insertTagItemTags.

private void insertTagItemTags(final Long id, final Set<String> tags, final Connection conn) throws SQLException {
    final Object[][] params = new Object[tags.size()][];
    final Iterator<String> iter = tags.iterator();
    int index = 0;
    while (iter.hasNext()) {
        params[index++] = ImmutableList.of(id, iter.next()).toArray();
    }
    new QueryRunner().batch(conn, SQL_INSERT_TAG_ITEM_TAGS, params);
}
Also used : QueryRunner(org.apache.commons.dbutils.QueryRunner)

Example 9 with QueryRunner

use of org.apache.commons.dbutils.QueryRunner in project metacat by Netflix.

the class MySqlTagService method delete.

@Override
public Void delete(final QualifiedName name, final boolean updateUserMetadata) {
    try {
        final Connection conn = getDataSource().getConnection();
        try {
            new QueryRunner().update(conn, SQL_DELETE_TAG_ITEM_TAGS_BY_NAME, name.toString());
            new QueryRunner().update(conn, SQL_DELETE_TAG_ITEM, name.toString());
            if (updateUserMetadata) {
                // Set the tags in user metadata
                final Map<String, Set<String>> data = Maps.newHashMap();
                data.put(NAME_TAGS, Sets.newHashSet());
                userMetadataService.saveDefinitionMetadata(name, "admin", Optional.of(metacatJson.toJsonObject(data)), true);
            }
            conn.commit();
        } catch (SQLException e) {
            conn.rollback();
            throw e;
        } finally {
            conn.close();
        }
    } catch (SQLException e) {
        final String message = String.format("Failed to delete all tags for name %s", name);
        log.error(message, e);
        throw new UserMetadataServiceException(message, e);
    }
    return null;
}
Also used : UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) Set(java.util.Set) SQLException(java.sql.SQLException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner)

Example 10 with QueryRunner

use of org.apache.commons.dbutils.QueryRunner in project metacat by Netflix.

the class MySqlTagService method list.

/**
     * Returns the list of <code>QualifiedName</code> of items that are tagged by the
     * given <code>includeTags</code> and do not contain the given <code>excludeTags</code>.
     *
     * @param includeTags  include items that contain tags
     * @param excludeTags  include items that do not contain tags
     * @param sourceName   catalog/source name
     * @param databaseName database name
     * @param tableName    table name
     * @return list of qualified names of the items
     */
@Override
public List<QualifiedName> list(final Set<String> includeTags, final Set<String> excludeTags, final String sourceName, final String databaseName, final String tableName) {
    Set<String> includedNames = Sets.newHashSet();
    final Set<String> excludedNames = Sets.newHashSet();
    final Connection connection = DBUtil.getReadConnection(getDataSource());
    try {
        final QueryRunner runner = new QueryRunner();
        final String wildCardName = QualifiedName.toWildCardString(sourceName, databaseName, tableName);
        //Includes
        String query = String.format(QUERY_SEARCH, "in ('" + Joiner.on("','").skipNulls().join(includeTags) + "')");
        final Object[] params = { includeTags.size() == 0 ? 1 : 0, wildCardName == null ? 1 : 0, wildCardName };
        includedNames.addAll(runner.query(connection, query, new ColumnListHandler<>("name"), params));
        if (!excludeTags.isEmpty()) {
            //Excludes
            query = String.format(QUERY_SEARCH, "in ('" + Joiner.on("','").skipNulls().join(excludeTags) + "')");
            final Object[] eParams = { excludeTags.size() == 0 ? 1 : 0, wildCardName == null ? 1 : 0, wildCardName };
            excludedNames.addAll(runner.query(connection, query, new ColumnListHandler<>("name"), eParams));
        }
    } catch (SQLException e) {
        final String message = String.format("Failed getting the list of qualified names for tags %s", includeTags);
        log.error(message, e);
        throw new UserMetadataServiceException(message, e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    if (!excludeTags.isEmpty()) {
        includedNames = Sets.difference(includedNames, excludedNames);
    }
    return includedNames.stream().map(s -> QualifiedName.fromString(s, false)).collect(Collectors.toList());
}
Also used : Lookup(com.netflix.metacat.common.server.model.Lookup) UserMetadataService(com.netflix.metacat.common.server.usermetadata.UserMetadataService) Connection(java.sql.Connection) TagItem(com.netflix.metacat.common.server.model.TagItem) SQLException(java.sql.SQLException) TagService(com.netflix.metacat.common.server.usermetadata.TagService) ImmutableList(com.google.common.collect.ImmutableList) ScalarHandler(org.apache.commons.dbutils.handlers.ScalarHandler) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) LookupService(com.netflix.metacat.common.server.usermetadata.LookupService) Config(com.netflix.metacat.common.server.properties.Config) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) QueryRunner(org.apache.commons.dbutils.QueryRunner) Iterator(java.util.Iterator) MetacatJson(com.netflix.metacat.common.json.MetacatJson) BeanHandler(org.apache.commons.dbutils.handlers.BeanHandler) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) SQLException(java.sql.SQLException) Connection(java.sql.Connection) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) QueryRunner(org.apache.commons.dbutils.QueryRunner)

Aggregations

QueryRunner (org.apache.commons.dbutils.QueryRunner)48 SQLException (java.sql.SQLException)26 Connection (java.sql.Connection)25 UserMetadataServiceException (com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException)17 DataSource (javax.sql.DataSource)15 QualifiedName (com.netflix.metacat.common.QualifiedName)14 List (java.util.List)13 Map (java.util.Map)13 ResultSetHandler (org.apache.commons.dbutils.ResultSetHandler)13 Maps (com.google.common.collect.Maps)12 DataSourceManager (com.netflix.metacat.common.server.util.DataSourceManager)12 Date (java.util.Date)12 Slf4j (lombok.extern.slf4j.Slf4j)12 Joiner (com.google.common.base.Joiner)11 Lists (com.google.common.collect.Lists)11 Collectors (java.util.stream.Collectors)11 Nonnull (javax.annotation.Nonnull)11 Strings (com.google.common.base.Strings)10 Nullable (javax.annotation.Nullable)10 ObjectNode (com.fasterxml.jackson.databind.node.ObjectNode)7