Examples with DefinitionMetadataDto com.netflix.metacat.common.dto.DefinitionMetadataDto used on opensource projects

Example 1 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MetadataService method cleanUpObsoleteDefinitionMetadata.

/**
 * Deletes definition metadata of tables/views/partitions that have been deleted already.
 */
public void cleanUpObsoleteDefinitionMetadata() {
    log.info("Start deleting obsolete definition metadata");
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    List<DefinitionMetadataDto> dtos = null;
    int offset = 0;
    final int limit = 10000;
    final ThreadServiceManager threadServiceManager = new ThreadServiceManager(registry, 10, 20000, "definition");
    final ListeningExecutorService service = threadServiceManager.getExecutor();
    int totalDeletes = 0;
    while (offset == 0 || dtos.size() == limit) {
        dtos = userMetadataService.searchDefinitionMetadata(null, null, null, null, "id", null, offset, limit);
        int deletes = 0;
        final List<ListenableFuture<Boolean>> futures = dtos.stream().map(dto -> service.submit(() -> deleteDefinitionMetadata(dto.getName(), false, metacatRequestContext))).collect(Collectors.toList());
        try {
            deletes = Futures.transform(Futures.successfulAsList(futures), (Function<List<Boolean>, Integer>) input -> (int) (input != null ? input.stream().filter(b -> b != null && b).count() : 0)).get();
        } catch (Exception e) {
            log.warn("Failed deleting obsolete definition metadata for offset {}.", offset, e);
        }
        totalDeletes += deletes;
        offset += limit - deletes;
    }
    threadServiceManager.stop();
    log.info("End deleting obsolete definition metadata. Deleted {} number of definition metadatas", totalDeletes);
}

Example 2 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MysqlUserMetadataService method searchDefinitionMetadata.

@Override
@Transactional(readOnly = true)
public List<DefinitionMetadataDto> searchDefinitionMetadata(@Nullable final Set<String> propertyNames, @Nullable final String type, @Nullable final String name, @Nullable final HasMetadata holder, @Nullable final String sortBy, @Nullable final String sortOrder, @Nullable final Integer offset, @Nullable final Integer limit) {
    final List<DefinitionMetadataDto> result = Lists.newArrayList();
    final SearchMetadataQuery queryObj = new SearchMetadataQuery(SQL.SEARCH_DEFINITION_METADATAS).buildSearchMetadataQuery(propertyNames, type, name, sortBy, sortOrder, offset, limit);
    try {
        // Handler for reading the result set
        final ResultSetExtractor<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                final String data = rs.getString("data");
                final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
                final QualifiedName metadataName = QualifiedName.fromString(definitionName);
                definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
                // Apply business logic
                final ObjectNode node = metacatJson.parseJsonObject(data);
                this.metadataInterceptor.onRead(this, metadataName, node, GetMetadataInterceptorParameters.builder().hasMetadata(holder).build());
                definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
                result.add(definitionMetadataDto);
            }
            return null;
        };
        jdbcTemplate.query(queryObj.getSearchQuery().toString(), queryObj.getSearchParamList().toArray(), handler);
    } catch (Exception e) {
        log.error("Failed to search definition data", e);
        throw new UserMetadataServiceException("Failed to search definition data", e);
    }
    return result;
}

Example 3 with DefinitionMetadataDto

use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.

the class MysqlUserMetadataService method searchDefinitionMetadatas.

@Override
public List<DefinitionMetadataDto> searchDefinitionMetadatas(final Set<String> propertyNames, final String type, final String name, final String sortBy, final String sortOrder, final Integer offset, final Integer limit) {
    final List<DefinitionMetadataDto> result = Lists.newArrayList();
    final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATAS);
    final List<Object> paramList = Lists.newArrayList();
    if (type != null) {
        String typeRegex = null;
        switch(type) {
            case "database":
                typeRegex = "^[^/]*/[^/]*$";
                break;
            case "table":
                typeRegex = "^[^/]*/[^/]*/[^/]*$";
                break;
            case "partition":
                typeRegex = "^[^/]*/[^/]*/[^/]*/.*$";
                break;
            default:
        }
        if (typeRegex != null) {
            query.append(" and name rlike ?");
            paramList.add(typeRegex);
        }
    }
    if (propertyNames != null && !propertyNames.isEmpty()) {
        propertyNames.forEach(propertyName -> {
            query.append(" and data like ?");
            paramList.add("%\"" + propertyName + "\":%");
        });
    }
    if (!Strings.isNullOrEmpty(name)) {
        query.append(" and name like ?");
        paramList.add(name);
    }
    if (!Strings.isNullOrEmpty(sortBy)) {
        query.append(" order by ").append(sortBy);
        if (!Strings.isNullOrEmpty(sortOrder)) {
            query.append(" ").append(sortOrder);
        }
    }
    if (limit != null) {
        query.append(" limit ");
        if (offset != null) {
            query.append(offset).append(",");
        }
        query.append(limit);
    }
    final Object[] params = new Object[paramList.size()];
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        // Handler for reading the result set
        final ResultSetHandler<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                final String data = rs.getString("data");
                final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
                definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
                definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
                result.add(definitionMetadataDto);
            }
            return null;
        };
        new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to get definition data", e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}