Search in sources :

Example 1 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MetadataService method cleanUpObsoleteDefinitionMetadata.

/**
 * Deletes definition metadata of tables/views/partitions that have been deleted already.
 */
public void cleanUpObsoleteDefinitionMetadata() {
    log.info("Start deleting obsolete definition metadata");
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    List<DefinitionMetadataDto> dtos = null;
    int offset = 0;
    final int limit = 10000;
    final ThreadServiceManager threadServiceManager = new ThreadServiceManager(registry, 10, 20000, "definition");
    final ListeningExecutorService service = threadServiceManager.getExecutor();
    int totalDeletes = 0;
    while (offset == 0 || dtos.size() == limit) {
        dtos = userMetadataService.searchDefinitionMetadata(null, null, null, null, "id", null, offset, limit);
        int deletes = 0;
        final List<ListenableFuture<Boolean>> futures = dtos.stream().map(dto -> service.submit(() -> deleteDefinitionMetadata(dto.getName(), false, metacatRequestContext))).collect(Collectors.toList());
        try {
            deletes = Futures.transform(Futures.successfulAsList(futures), (Function<List<Boolean>, Integer>) input -> (int) (input != null ? input.stream().filter(b -> b != null && b).count() : 0)).get();
        } catch (Exception e) {
            log.warn("Failed deleting obsolete definition metadata for offset {}.", offset, e);
        }
        totalDeletes += deletes;
        offset += limit - deletes;
    }
    threadServiceManager.stop();
    log.info("End deleting obsolete definition metadata. Deleted {} number of definition metadatas", totalDeletes);
}
Also used : UserMetadataService(com.netflix.metacat.common.server.usermetadata.UserMetadataService) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Getter(lombok.Getter) MetacatContextManager(com.netflix.metacat.common.server.util.MetacatContextManager) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) TagService(com.netflix.metacat.common.server.usermetadata.TagService) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) BaseDto(com.netflix.metacat.common.dto.BaseDto) Map(java.util.Map) MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) NotFoundException(com.netflix.metacat.common.server.connectors.exception.NotFoundException) Config(com.netflix.metacat.common.server.properties.Config) Metrics(com.netflix.metacat.common.server.monitoring.Metrics) Function(com.google.common.base.Function) Collection(java.util.Collection) DateTime(org.joda.time.DateTime) QualifiedName(com.netflix.metacat.common.QualifiedName) Collectors(java.util.stream.Collectors) Futures(com.google.common.util.concurrent.Futures) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Stream(java.util.stream.Stream) Registry(com.netflix.spectator.api.Registry) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) ThreadServiceManager(com.netflix.metacat.common.server.util.ThreadServiceManager) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) List(java.util.List) NotFoundException(com.netflix.metacat.common.server.connectors.exception.NotFoundException)

Example 2 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MysqlUserMetadataService method searchDefinitionMetadata.

@Override
@Transactional(readOnly = true)
public List<DefinitionMetadataDto> searchDefinitionMetadata(@Nullable final Set<String> propertyNames, @Nullable final String type, @Nullable final String name, @Nullable final HasMetadata holder, @Nullable final String sortBy, @Nullable final String sortOrder, @Nullable final Integer offset, @Nullable final Integer limit) {
    final List<DefinitionMetadataDto> result = Lists.newArrayList();
    final SearchMetadataQuery queryObj = new SearchMetadataQuery(SQL.SEARCH_DEFINITION_METADATAS).buildSearchMetadataQuery(propertyNames, type, name, sortBy, sortOrder, offset, limit);
    try {
        // Handler for reading the result set
        final ResultSetExtractor<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                final String data = rs.getString("data");
                final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
                final QualifiedName metadataName = QualifiedName.fromString(definitionName);
                definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
                // Apply business logic
                final ObjectNode node = metacatJson.parseJsonObject(data);
                this.metadataInterceptor.onRead(this, metadataName, node, GetMetadataInterceptorParameters.builder().hasMetadata(holder).build());
                definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
                result.add(definitionMetadataDto);
            }
            return null;
        };
        jdbcTemplate.query(queryObj.getSearchQuery().toString(), queryObj.getSearchParamList().toArray(), handler);
    } catch (Exception e) {
        log.error("Failed to search definition data", e);
        throw new UserMetadataServiceException("Failed to search definition data", e);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) InvalidMetadataException(com.netflix.metacat.common.server.connectors.exception.InvalidMetadataException) MetacatJson(com.netflix.metacat.common.json.MetacatJson) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) SqlParameterValue(org.springframework.jdbc.core.SqlParameterValue) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Data(lombok.Data) GetMetadataInterceptorParameters(com.netflix.metacat.common.server.usermetadata.GetMetadataInterceptorParameters) MetadataInterceptor(com.netflix.metacat.common.server.usermetadata.MetadataInterceptor) Optional(java.util.Optional) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) ResultSetExtractor(org.springframework.jdbc.core.ResultSetExtractor) Joiner(com.google.common.base.Joiner) Transactional(org.springframework.transaction.annotation.Transactional) Types(java.sql.Types) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) QualifiedName(com.netflix.metacat.common.QualifiedName) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) InvalidMetadataException(com.netflix.metacat.common.server.connectors.exception.InvalidMetadataException) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Transactional(org.springframework.transaction.annotation.Transactional)

Example 3 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MysqlUserMetadataService method searchDefinitionMetadatas.

@Override
public List<DefinitionMetadataDto> searchDefinitionMetadatas(final Set<String> propertyNames, final String type, final String name, final String sortBy, final String sortOrder, final Integer offset, final Integer limit) {
    final List<DefinitionMetadataDto> result = Lists.newArrayList();
    final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATAS);
    final List<Object> paramList = Lists.newArrayList();
    if (type != null) {
        String typeRegex = null;
        switch(type) {
            case "database":
                typeRegex = "^[^/]*/[^/]*$";
                break;
            case "table":
                typeRegex = "^[^/]*/[^/]*/[^/]*$";
                break;
            case "partition":
                typeRegex = "^[^/]*/[^/]*/[^/]*/.*$";
                break;
            default:
        }
        if (typeRegex != null) {
            query.append(" and name rlike ?");
            paramList.add(typeRegex);
        }
    }
    if (propertyNames != null && !propertyNames.isEmpty()) {
        propertyNames.forEach(propertyName -> {
            query.append(" and data like ?");
            paramList.add("%\"" + propertyName + "\":%");
        });
    }
    if (!Strings.isNullOrEmpty(name)) {
        query.append(" and name like ?");
        paramList.add(name);
    }
    if (!Strings.isNullOrEmpty(sortBy)) {
        query.append(" order by ").append(sortBy);
        if (!Strings.isNullOrEmpty(sortOrder)) {
            query.append(" ").append(sortOrder);
        }
    }
    if (limit != null) {
        query.append(" limit ");
        if (offset != null) {
            query.append(offset).append(",");
        }
        query.append(limit);
    }
    final Object[] params = new Object[paramList.size()];
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        // Handler for reading the result set
        final ResultSetHandler<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                final String data = rs.getString("data");
                final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
                definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
                definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
                result.add(definitionMetadataDto);
            }
            return null;
        };
        new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to get definition data", e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}
Also used : Arrays(java.util.Arrays) Connection(java.sql.Connection) URL(java.net.URL) Date(java.util.Date) HasDataMetadata(com.netflix.metacat.common.dto.HasDataMetadata) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto) Strings(com.google.common.base.Strings) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ResultSet(java.sql.ResultSet) Map(java.util.Map) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) DataSource(javax.sql.DataSource) Config(com.netflix.metacat.common.server.properties.Config) BaseUserMetadataService(com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) Charsets(com.google.common.base.Charsets) DataSourceManager(com.netflix.metacat.common.server.util.DataSourceManager) Properties(java.util.Properties) QueryRunner(org.apache.commons.dbutils.QueryRunner) MetacatJson(com.netflix.metacat.common.json.MetacatJson) Files(java.nio.file.Files) HasDefinitionMetadata(com.netflix.metacat.common.dto.HasDefinitionMetadata) Set(java.util.Set) QualifiedName(com.netflix.metacat.common.QualifiedName) Reader(java.io.Reader) PreparedStatement(java.sql.PreparedStatement) Maps(com.google.common.collect.Maps) MetacatJsonException(com.netflix.metacat.common.json.MetacatJsonException) Collectors(java.util.stream.Collectors) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) DBUtil(com.netflix.metacat.common.server.util.DBUtil) Paths(java.nio.file.Paths) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ColumnListHandler(org.apache.commons.dbutils.handlers.ColumnListHandler) Collections(java.util.Collections) ResultSetHandler(org.apache.commons.dbutils.ResultSetHandler) FileSystems(java.nio.file.FileSystems) Joiner(com.google.common.base.Joiner) UserMetadataServiceException(com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException) SQLException(java.sql.SQLException) Connection(java.sql.Connection) QueryRunner(org.apache.commons.dbutils.QueryRunner) DefinitionMetadataDto(com.netflix.metacat.common.dto.DefinitionMetadataDto)

Aggregations

Strings (com.google.common.base.Strings)3 Lists (com.google.common.collect.Lists)3 QualifiedName (com.netflix.metacat.common.QualifiedName)3 DefinitionMetadataDto (com.netflix.metacat.common.dto.DefinitionMetadataDto)3 Config (com.netflix.metacat.common.server.properties.Config)3 List (java.util.List)3 Map (java.util.Map)3 Collectors (java.util.stream.Collectors)3 Slf4j (lombok.extern.slf4j.Slf4j)3 ObjectNode (com.fasterxml.jackson.databind.node.ObjectNode)2 Joiner (com.google.common.base.Joiner)2 Maps (com.google.common.collect.Maps)2 HasDataMetadata (com.netflix.metacat.common.dto.HasDataMetadata)2 HasDefinitionMetadata (com.netflix.metacat.common.dto.HasDefinitionMetadata)2 HasMetadata (com.netflix.metacat.common.dto.HasMetadata)2 MetacatJson (com.netflix.metacat.common.json.MetacatJson)2 MetacatJsonException (com.netflix.metacat.common.json.MetacatJsonException)2 BaseUserMetadataService (com.netflix.metacat.common.server.usermetadata.BaseUserMetadataService)2 UserMetadataServiceException (com.netflix.metacat.common.server.usermetadata.UserMetadataServiceException)2 Arrays (java.util.Arrays)2