use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.
the class MetadataService method cleanUpObsoleteDefinitionMetadata.
/**
* Deletes definition metadata of tables/views/partitions that have been deleted already.
*/
public void cleanUpObsoleteDefinitionMetadata() {
log.info("Start deleting obsolete definition metadata");
final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
List<DefinitionMetadataDto> dtos = null;
int offset = 0;
final int limit = 10000;
final ThreadServiceManager threadServiceManager = new ThreadServiceManager(registry, 10, 20000, "definition");
final ListeningExecutorService service = threadServiceManager.getExecutor();
int totalDeletes = 0;
while (offset == 0 || dtos.size() == limit) {
dtos = userMetadataService.searchDefinitionMetadata(null, null, null, null, "id", null, offset, limit);
int deletes = 0;
final List<ListenableFuture<Boolean>> futures = dtos.stream().map(dto -> service.submit(() -> deleteDefinitionMetadata(dto.getName(), false, metacatRequestContext))).collect(Collectors.toList());
try {
deletes = Futures.transform(Futures.successfulAsList(futures), (Function<List<Boolean>, Integer>) input -> (int) (input != null ? input.stream().filter(b -> b != null && b).count() : 0)).get();
} catch (Exception e) {
log.warn("Failed deleting obsolete definition metadata for offset {}.", offset, e);
}
totalDeletes += deletes;
offset += limit - deletes;
}
threadServiceManager.stop();
log.info("End deleting obsolete definition metadata. Deleted {} number of definition metadatas", totalDeletes);
}
use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.
the class MysqlUserMetadataService method searchDefinitionMetadata.
@Override
@Transactional(readOnly = true)
public List<DefinitionMetadataDto> searchDefinitionMetadata(@Nullable final Set<String> propertyNames, @Nullable final String type, @Nullable final String name, @Nullable final HasMetadata holder, @Nullable final String sortBy, @Nullable final String sortOrder, @Nullable final Integer offset, @Nullable final Integer limit) {
final List<DefinitionMetadataDto> result = Lists.newArrayList();
final SearchMetadataQuery queryObj = new SearchMetadataQuery(SQL.SEARCH_DEFINITION_METADATAS).buildSearchMetadataQuery(propertyNames, type, name, sortBy, sortOrder, offset, limit);
try {
// Handler for reading the result set
final ResultSetExtractor<Void> handler = rs -> {
while (rs.next()) {
final String definitionName = rs.getString("name");
final String data = rs.getString("data");
final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
final QualifiedName metadataName = QualifiedName.fromString(definitionName);
definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
// Apply business logic
final ObjectNode node = metacatJson.parseJsonObject(data);
this.metadataInterceptor.onRead(this, metadataName, node, GetMetadataInterceptorParameters.builder().hasMetadata(holder).build());
definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
result.add(definitionMetadataDto);
}
return null;
};
jdbcTemplate.query(queryObj.getSearchQuery().toString(), queryObj.getSearchParamList().toArray(), handler);
} catch (Exception e) {
log.error("Failed to search definition data", e);
throw new UserMetadataServiceException("Failed to search definition data", e);
}
return result;
}
use of com.netflix.metacat.common.dto.DefinitionMetadataDto in project metacat by Netflix.
the class MysqlUserMetadataService method searchDefinitionMetadatas.
@Override
public List<DefinitionMetadataDto> searchDefinitionMetadatas(final Set<String> propertyNames, final String type, final String name, final String sortBy, final String sortOrder, final Integer offset, final Integer limit) {
final List<DefinitionMetadataDto> result = Lists.newArrayList();
final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATAS);
final List<Object> paramList = Lists.newArrayList();
if (type != null) {
String typeRegex = null;
switch(type) {
case "database":
typeRegex = "^[^/]*/[^/]*$";
break;
case "table":
typeRegex = "^[^/]*/[^/]*/[^/]*$";
break;
case "partition":
typeRegex = "^[^/]*/[^/]*/[^/]*/.*$";
break;
default:
}
if (typeRegex != null) {
query.append(" and name rlike ?");
paramList.add(typeRegex);
}
}
if (propertyNames != null && !propertyNames.isEmpty()) {
propertyNames.forEach(propertyName -> {
query.append(" and data like ?");
paramList.add("%\"" + propertyName + "\":%");
});
}
if (!Strings.isNullOrEmpty(name)) {
query.append(" and name like ?");
paramList.add(name);
}
if (!Strings.isNullOrEmpty(sortBy)) {
query.append(" order by ").append(sortBy);
if (!Strings.isNullOrEmpty(sortOrder)) {
query.append(" ").append(sortOrder);
}
}
if (limit != null) {
query.append(" limit ");
if (offset != null) {
query.append(offset).append(",");
}
query.append(limit);
}
final Object[] params = new Object[paramList.size()];
final Connection connection = DBUtil.getReadConnection(poolingDataSource);
try {
// Handler for reading the result set
final ResultSetHandler<Void> handler = rs -> {
while (rs.next()) {
final String definitionName = rs.getString("name");
final String data = rs.getString("data");
final DefinitionMetadataDto definitionMetadataDto = new DefinitionMetadataDto();
definitionMetadataDto.setName(QualifiedName.fromString(definitionName));
definitionMetadataDto.setDefinitionMetadata(metacatJson.parseJsonObject(data));
result.add(definitionMetadataDto);
}
return null;
};
new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
} catch (SQLException e) {
log.error("Sql exception", e);
throw new UserMetadataServiceException("Failed to get definition data", e);
} finally {
DBUtil.closeReadConnection(connection);
}
return result;
}
Aggregations