Search in sources :

Example 11 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MetacatV1Resource method createDatabase.

@Override
public void createDatabase(final String catalogName, final String databaseName, final DatabaseCreateRequestDto databaseCreateRequestDto) {
    final QualifiedName name = requestWrapper.qualifyName(() -> QualifiedName.ofDatabase(catalogName, databaseName));
    requestWrapper.processRequest(name, "createDatabase", () -> {
        final DatabaseDto newDto = new DatabaseDto();
        newDto.setName(name);
        if (databaseCreateRequestDto != null) {
            newDto.setDefinitionMetadata(databaseCreateRequestDto.getDefinitionMetadata());
        }
        databaseService.create(name, newDto);
        return null;
    });
}
Also used : QualifiedName(com.netflix.metacat.common.QualifiedName) DatabaseDto(com.netflix.metacat.common.dto.DatabaseDto)

Example 12 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class PartitionV1Resource method savePartitions.

/**
     * {@inheritDoc}
     */
@Override
public PartitionsSaveResponseDto savePartitions(final String catalogName, final String databaseName, final String tableName, final String viewName, final PartitionsSaveRequestDto partitionsSaveRequestDto) {
    final QualifiedName name = requestWrapper.qualifyName(() -> QualifiedName.ofView(catalogName, databaseName, tableName, viewName));
    return requestWrapper.processRequest(name, "saveMViewPartition", () -> {
        final PartitionsSaveResponseDto result;
        if (partitionsSaveRequestDto == null || partitionsSaveRequestDto.getPartitions() == null || partitionsSaveRequestDto.getPartitions().isEmpty()) {
            result = new PartitionsSaveResponseDto();
        } else {
            result = mViewService.savePartitions(name, partitionsSaveRequestDto, true);
            // This metadata is actually for the view, if it is present update that
            if (partitionsSaveRequestDto.getDefinitionMetadata() != null || partitionsSaveRequestDto.getDataMetadata() != null) {
                final TableDto dto = v1.getMView(catalogName, databaseName, tableName, viewName);
                dto.setDefinitionMetadata(partitionsSaveRequestDto.getDefinitionMetadata());
                dto.setDataMetadata(partitionsSaveRequestDto.getDataMetadata());
                v1.updateMView(catalogName, databaseName, tableName, viewName, dto);
            }
        }
        return result;
    });
}
Also used : QualifiedName(com.netflix.metacat.common.QualifiedName) TableDto(com.netflix.metacat.common.dto.TableDto) PartitionsSaveResponseDto(com.netflix.metacat.common.dto.PartitionsSaveResponseDto)

Example 13 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MysqlUserMetadataService method _deleteDefinitionMetadatas.

@SuppressWarnings("checkstyle:methodname")
private Void _deleteDefinitionMetadatas(final Connection conn, @Nullable final List<QualifiedName> names) throws SQLException {
    if (names != null && !names.isEmpty()) {
        final List<String> paramVariables = names.stream().map(s -> "?").collect(Collectors.toList());
        final String[] aNames = names.stream().map(QualifiedName::toString).toArray(String[]::new);
        new QueryRunner().update(conn, String.format(SQL.DELETE_DEFINITION_METADATA, Joiner.on(",").skipNulls().join(paramVariables)), (Object[]) aNames);
    }
    return null;
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) QueryRunner(org.apache.commons.dbutils.QueryRunner)

Example 14 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorFastPartitionService method getPartitionCount.

/**
     * Number of partitions for the given table.
     *
     * @param tableName tableName
     * @return Number of partitions
     */
@Override
public int getPartitionCount(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName) {
    final long start = registry.clock().monotonicTime();
    final Map<String, String> tags = new HashMap<String, String>();
    tags.put("request", HiveMetrics.getPartitionCount.name());
    final Integer result;
    final DataSource dataSource = DataSourceManager.get().get(catalogName);
    try (Connection conn = dataSource.getConnection()) {
        // Handler for reading the result set
        final ResultSetHandler<Integer> handler = rs -> {
            int count = 0;
            while (rs.next()) {
                count = rs.getInt("count");
            }
            return count;
        };
        result = new QueryRunner().query(conn, SQL_GET_PARTITION_COUNT, handler, tableName.getDatabaseName(), tableName.getTableName());
    } catch (SQLException e) {
        throw new ConnectorException("getPartitionCount", e);
    } finally {
        final long duration = registry.clock().monotonicTime() - start;
        log.debug("### Time taken to complete getPartitionCount is {} ms", duration);
        this.registry.timer(requestTimerId.withTags(tags)).record(duration, TimeUnit.MILLISECONDS);
    }
    return result;
}
Also used : Connection(java.sql.Connection) PartitionKeyParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionKeyParserEval) Date(java.util.Date) PartitionFilterGenerator(com.netflix.metacat.connector.hive.util.PartitionFilterGenerator) PartitionParamParserEval(com.netflix.metacat.common.server.partition.visitor.PartitionParamParserEval) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) Map(java.util.Map) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) QueryRunner(org.apache.commons.dbutils.QueryRunner) NonNull(lombok.NonNull) Collection(java.util.Collection) Pageable(com.netflix.metacat.common.dto.Pageable) QualifiedName(com.netflix.metacat.common.QualifiedName) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) HiveMetrics(com.netflix.metacat.connector.hive.monitoring.HiveMetrics) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) Joiner(com.google.common.base.Joiner) Sort(com.netflix.metacat.common.dto.Sort) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AuditInfo(com.netflix.metacat.common.server.connectors.model.AuditInfo) HashMap(java.util.HashMap) Id(com.netflix.spectator.api.Id) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) DataSource(javax.sql.DataSource) PartitionParser(com.netflix.metacat.common.server.partition.parser.PartitionParser) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) PartitionDetail(com.netflix.metacat.connector.hive.util.PartitionDetail) Functions(com.google.common.base.Functions) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Throwables(com.google.common.base.Throwables) Maps(com.google.common.collect.Maps) FilterPartition(com.netflix.metacat.common.server.partition.util.FilterPartition) TimeUnit(java.util.concurrent.TimeUnit) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Futures(com.google.common.util.concurrent.Futures) StringReader(java.io.StringReader) Registry(com.netflix.spectator.api.Registry) PartitionListRequest(com.netflix.metacat.common.server.connectors.model.PartitionListRequest) HashMap(java.util.HashMap) SQLException(java.sql.SQLException) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DataSource(javax.sql.DataSource)

Example 15 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorTableService method list.

/**
     * {@inheritDoc}.
     */
@Override
public List<TableInfo> list(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName name, @Nullable final QualifiedName prefix, @Nullable final Sort sort, @Nullable final Pageable pageable) {
    try {
        final List<TableInfo> tableInfos = Lists.newArrayList();
        for (String tableName : metacatHiveClient.getAllTables(name.getDatabaseName())) {
            final QualifiedName qualifiedName = QualifiedName.ofDatabase(name.getCatalogName(), tableName);
            if (!qualifiedName.toString().startsWith(prefix.toString())) {
                continue;
            }
            final Table table = metacatHiveClient.getTableByName(name.getDatabaseName(), tableName);
            tableInfos.add(hiveMetacatConverters.toTableInfo(name, table));
        }
        //supporting sort by name only
        if (sort != null) {
            ConnectorUtils.sort(tableInfos, sort, Comparator.comparing(p -> p.getName().getTableName()));
        }
        return ConnectorUtils.paginate(tableInfos, pageable);
    } catch (MetaException exception) {
        throw new DatabaseNotFoundException(name, exception);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed list hive table %s", name), exception);
    }
}
Also used : MetaException(org.apache.hadoop.hive.metastore.api.MetaException) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) DatabaseNotFoundException(com.netflix.metacat.common.server.connectors.exception.DatabaseNotFoundException) AlreadyExistsException(org.apache.hadoop.hive.metastore.api.AlreadyExistsException) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) ConnectorTableService(com.netflix.metacat.common.server.connectors.ConnectorTableService) FieldInfo(com.netflix.metacat.common.server.connectors.model.FieldInfo) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) StorageInfo(com.netflix.metacat.common.server.connectors.model.StorageInfo) Named(javax.inject.Named) HiveConnectorInfoConverter(com.netflix.metacat.connector.hive.converters.HiveConnectorInfoConverter) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) ImmutableMap(com.google.common.collect.ImmutableMap) NonNull(lombok.NonNull) Pageable(com.netflix.metacat.common.dto.Pageable) TException(org.apache.thrift.TException) QualifiedName(com.netflix.metacat.common.QualifiedName) InvalidObjectException(org.apache.hadoop.hive.metastore.api.InvalidObjectException) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) Maps(com.google.common.collect.Maps) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) List(java.util.List) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) TableAlreadyExistsException(com.netflix.metacat.common.server.connectors.exception.TableAlreadyExistsException) TableType(org.apache.hadoop.hive.metastore.TableType) ConnectorUtils(com.netflix.metacat.common.server.connectors.ConnectorUtils) Comparator(java.util.Comparator) NoSuchObjectException(org.apache.hadoop.hive.metastore.api.NoSuchObjectException) Sort(com.netflix.metacat.common.dto.Sort) TException(org.apache.thrift.TException) Table(org.apache.hadoop.hive.metastore.api.Table) QualifiedName(com.netflix.metacat.common.QualifiedName) DatabaseNotFoundException(com.netflix.metacat.common.server.connectors.exception.DatabaseNotFoundException) ConnectorException(com.netflix.metacat.common.server.connectors.exception.ConnectorException) TableInfo(com.netflix.metacat.common.server.connectors.model.TableInfo) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) InvalidMetaException(com.netflix.metacat.common.server.connectors.exception.InvalidMetaException)

Aggregations

QualifiedName (com.netflix.metacat.common.QualifiedName)85 List (java.util.List)31 Lists (com.google.common.collect.Lists)25 Collectors (java.util.stream.Collectors)24 Map (java.util.Map)23 Nonnull (javax.annotation.Nonnull)23 Strings (com.google.common.base.Strings)20 Slf4j (lombok.extern.slf4j.Slf4j)20 MetacatRequestContext (com.netflix.metacat.common.MetacatRequestContext)19 TableDto (com.netflix.metacat.common.dto.TableDto)19 ConnectorContext (com.netflix.metacat.common.server.connectors.ConnectorContext)17 Pageable (com.netflix.metacat.common.dto.Pageable)16 Sort (com.netflix.metacat.common.dto.Sort)16 Maps (com.google.common.collect.Maps)14 Connection (java.sql.Connection)14 SQLException (java.sql.SQLException)14 NonNull (lombok.NonNull)14 TableNotFoundException (com.netflix.metacat.common.server.connectors.exception.TableNotFoundException)13 Nullable (javax.annotation.Nullable)13 DatabaseNotFoundException (com.netflix.metacat.common.server.connectors.exception.DatabaseNotFoundException)12