Search in sources :

Example 81 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MysqlUserMetadataService method deleteMetadatas.

@Override
public void deleteMetadatas(final String userId, final List<HasMetadata> holders) {
    try {
        final Connection conn = poolingDataSource.getConnection();
        try {
            final List<List<HasMetadata>> subLists = Lists.partition(holders, config.getUserMetadataMaxInClauseItems());
            for (List<HasMetadata> hasMetadatas : subLists) {
                final List<QualifiedName> names = hasMetadatas.stream().filter(m -> m instanceof HasDefinitionMetadata).map(m -> ((HasDefinitionMetadata) m).getDefinitionName()).collect(Collectors.toList());
                if (!names.isEmpty()) {
                    _deleteDefinitionMetadatas(conn, names);
                }
                if (config.canSoftDeleteDataMetadata()) {
                    final List<String> uris = hasMetadatas.stream().filter(m -> m instanceof HasDataMetadata && ((HasDataMetadata) m).isDataExternal()).map(m -> ((HasDataMetadata) m).getDataUri()).collect(Collectors.toList());
                    if (!uris.isEmpty()) {
                        _softDeleteDataMetadatas(conn, userId, uris);
                    }
                }
            }
            conn.commit();
        } catch (SQLException e) {
            conn.rollback();
            throw e;
        } finally {
            conn.close();
        }
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed deleting data metadata", e);
    }
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) SQLException(java.sql.SQLException) QualifiedName(com.netflix.metacat.common.QualifiedName) Connection(java.sql.Connection) List(java.util.List) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata)

Example 82 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConvertersImpl method hiveToMetacatTable.

/**
     * {@inheritDoc}
     */
@Override
public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) {
    final TableDto dto = new TableDto();
    dto.setSerde(toStorageDto(table.getSd(), table.getOwner()));
    dto.setAudit(new AuditDto());
    dto.setName(name);
    if (table.isSetCreateTime()) {
        dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime()));
    }
    dto.setMetadata(table.getParameters());
    final List<FieldSchema> nonPartitionColumns = table.getSd().getCols();
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final List<FieldDto> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream().map(field -> this.hiveToMetacatField(field, false)).forEachOrdered(allFields::add);
    partitionColumns.stream().map(field -> this.hiveToMetacatField(field, true)).forEachOrdered(allFields::add);
    dto.setFields(allFields);
    return dto;
}
Also used : TableDto(com.netflix.metacat.common.dto.TableDto) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) Date(java.util.Date) HashMap(java.util.HashMap) DatabaseDto(com.netflix.metacat.common.dto.DatabaseDto) StorageDto(com.netflix.metacat.common.dto.StorageDto) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) Partition(org.apache.hadoop.hive.metastore.api.Partition) Warehouse(org.apache.hadoop.hive.metastore.Warehouse) LinkedHashMap(java.util.LinkedHashMap) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) AuditDto(com.netflix.metacat.common.dto.AuditDto) Splitter(com.google.common.base.Splitter) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Nullable(javax.annotation.Nullable) QualifiedName(com.netflix.metacat.common.QualifiedName) FieldDto(com.netflix.metacat.common.dto.FieldDto) Instant(java.time.Instant) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) List(java.util.List) PartitionDto(com.netflix.metacat.common.dto.PartitionDto) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Database(org.apache.hadoop.hive.metastore.api.Database) Collections(java.util.Collections) AuditDto(com.netflix.metacat.common.dto.AuditDto) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) TableDto(com.netflix.metacat.common.dto.TableDto) FieldDto(com.netflix.metacat.common.dto.FieldDto)

Example 83 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConvertersImpl method hiveToMetacatPartition.

/**
     * {@inheritDoc}
     */
@Override
public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) {
    final QualifiedName tableName = tableDto.getName();
    final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues()));
    final PartitionDto result = new PartitionDto();
    String owner = "";
    if (tableDto.getSerde() != null) {
        owner = tableDto.getSerde().getOwner();
    }
    result.setSerde(toStorageDto(partition.getSd(), owner));
    result.setMetadata(partition.getParameters());
    final AuditDto auditDto = new AuditDto();
    auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime()));
    auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime()));
    result.setAudit(auditDto);
    result.setName(partitionName);
    return result;
}
Also used : AuditDto(com.netflix.metacat.common.dto.AuditDto) QualifiedName(com.netflix.metacat.common.QualifiedName) PartitionDto(com.netflix.metacat.common.dto.PartitionDto)

Example 84 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConvertersImpl method metacatToHivePartition.

/**
     * {@inheritDoc}
     */
@Override
public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) {
    final Partition result = new Partition();
    final QualifiedName name = partitionDto.getName();
    final List<String> values = Lists.newArrayListWithCapacity(16);
    String databaseName = "";
    String tableName = "";
    if (name != null) {
        if (name.getPartitionName() != null) {
            for (String partialPartName : SLASH_SPLITTER.split(partitionDto.getName().getPartitionName())) {
                final List<String> nameValues = ImmutableList.copyOf(EQUAL_SPLITTER.split(partialPartName));
                if (nameValues.size() != 2) {
                    throw new IllegalStateException("Unrecognized partition name: " + partitionDto.getName());
                }
                final String value = nameValues.get(1);
                values.add(value);
            }
        }
        if (name.getDatabaseName() != null) {
            databaseName = name.getDatabaseName();
        }
        if (name.getTableName() != null) {
            tableName = name.getTableName();
        }
    }
    result.setValues(values);
    result.setDbName(databaseName);
    result.setTableName(tableName);
    Map<String, String> metadata = partitionDto.getMetadata();
    if (metadata == null) {
        metadata = Maps.newHashMap();
    }
    result.setParameters(metadata);
    result.setSd(fromStorageDto(partitionDto.getSerde()));
    final StorageDescriptor sd = result.getSd();
    if (tableDto != null) {
        if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty(sd.getSerdeInfo().getSerializationLib())) {
            sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib());
        }
        final List<FieldDto> fields = tableDto.getFields();
        if (fields == null) {
            sd.setCols(Collections.emptyList());
        } else {
            sd.setCols(fields.stream().filter(field -> !field.isPartition_key()).map(this::metacatToHiveField).collect(Collectors.toList()));
        }
    }
    final AuditDto auditDto = partitionDto.getAudit();
    if (auditDto != null) {
        if (auditDto.getCreatedDate() != null) {
            result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
        }
        if (auditDto.getLastModifiedDate() != null) {
            result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
        }
    }
    return result;
}
Also used : TableDto(com.netflix.metacat.common.dto.TableDto) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) Date(java.util.Date) HashMap(java.util.HashMap) DatabaseDto(com.netflix.metacat.common.dto.DatabaseDto) StorageDto(com.netflix.metacat.common.dto.StorageDto) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) Partition(org.apache.hadoop.hive.metastore.api.Partition) Warehouse(org.apache.hadoop.hive.metastore.Warehouse) LinkedHashMap(java.util.LinkedHashMap) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) AuditDto(com.netflix.metacat.common.dto.AuditDto) Splitter(com.google.common.base.Splitter) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) Nullable(javax.annotation.Nullable) QualifiedName(com.netflix.metacat.common.QualifiedName) FieldDto(com.netflix.metacat.common.dto.FieldDto) Instant(java.time.Instant) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) Table(org.apache.hadoop.hive.metastore.api.Table) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) List(java.util.List) PartitionDto(com.netflix.metacat.common.dto.PartitionDto) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Database(org.apache.hadoop.hive.metastore.api.Database) Collections(java.util.Collections) Partition(org.apache.hadoop.hive.metastore.api.Partition) AuditDto(com.netflix.metacat.common.dto.AuditDto) QualifiedName(com.netflix.metacat.common.QualifiedName) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) FieldDto(com.netflix.metacat.common.dto.FieldDto)

Example 85 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class CatalogThriftHiveMetastore method drop_partition_with_environment_context.

/**
     * {@inheritDoc}
     */
@Override
public boolean drop_partition_with_environment_context(final String dbName, final String tblName, final List<String> partVals, final boolean deleteData, @Nullable final EnvironmentContext environmentContext) throws TException {
    return requestWrapper("drop_partition_with_environment_context", new Object[] { dbName, tblName, partVals, deleteData, environmentContext }, () -> {
        final TableDto tableDto = getTableDto(dbName, tblName);
        final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals);
        final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName();
        if (deleteData) {
            log.warn("Ignoring command to delete data for {}/{}/{}/{}", partitionName);
        }
        partV1.deletePartitions(catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), ImmutableList.of(partitionName.getPartitionName()));
        return true;
    });
}
Also used : QualifiedName(com.netflix.metacat.common.QualifiedName) TableDto(com.netflix.metacat.common.dto.TableDto)

Aggregations

QualifiedName (com.netflix.metacat.common.QualifiedName)85 List (java.util.List)31 Lists (com.google.common.collect.Lists)25 Collectors (java.util.stream.Collectors)24 Map (java.util.Map)23 Nonnull (javax.annotation.Nonnull)23 Strings (com.google.common.base.Strings)20 Slf4j (lombok.extern.slf4j.Slf4j)20 MetacatRequestContext (com.netflix.metacat.common.MetacatRequestContext)19 TableDto (com.netflix.metacat.common.dto.TableDto)19 ConnectorContext (com.netflix.metacat.common.server.connectors.ConnectorContext)17 Pageable (com.netflix.metacat.common.dto.Pageable)16 Sort (com.netflix.metacat.common.dto.Sort)16 Maps (com.google.common.collect.Maps)14 Connection (java.sql.Connection)14 SQLException (java.sql.SQLException)14 NonNull (lombok.NonNull)14 TableNotFoundException (com.netflix.metacat.common.server.connectors.exception.TableNotFoundException)13 Nullable (javax.annotation.Nullable)13 DatabaseNotFoundException (com.netflix.metacat.common.server.connectors.exception.DatabaseNotFoundException)12