Examples with UserMetadataService com.netflix.metacat.common.server.usermetadata.UserMetadataService used on opensource projects

Example 1 with UserMetadataService

use of com.netflix.metacat.common.server.usermetadata.UserMetadataService in project metacat by Netflix.

the class MetadataService method cleanUpDeletedDataMetadata.

/**
 * Deletes all the data metadata marked for deletion.
 */
public void cleanUpDeletedDataMetadata() {
    // Get the data metadata that were marked deleted a number of days back
    // Check if the uri is being used
    // If uri is not used then delete the entry from data_metadata
    log.info("Start deleting data metadata");
    try {
        final DateTime priorTo = DateTime.now().minusDays(config.getDataMetadataDeleteMarkerLifetimeInDays());
        final int limit = 100000;
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        while (true) {
            final List<String> urisToDelete = userMetadataService.getDeletedDataMetadataUris(priorTo.toDate(), 0, limit);
            log.info("Count of deleted marked data metadata: {}", urisToDelete.size());
            if (urisToDelete.size() > 0) {
                final List<String> uris = urisToDelete.parallelStream().filter(uri -> !uri.contains("=")).map(userMetadataService::getDescendantDataUris).flatMap(Collection::stream).collect(Collectors.toList());
                uris.addAll(urisToDelete);
                log.info("Count of deleted marked data metadata (including descendants) : {}", uris.size());
                final List<List<String>> subListsUris = Lists.partition(uris, 1000);
                subListsUris.parallelStream().forEach(subUris -> {
                    MetacatContextManager.setContext(metacatRequestContext);
                    final Map<String, List<QualifiedName>> uriPartitionQualifiedNames = partitionService.getQualifiedNames(subUris, false);
                    final Map<String, List<QualifiedName>> uriTableQualifiedNames = tableService.getQualifiedNames(subUris, false);
                    final Map<String, List<QualifiedName>> uriQualifiedNames = Stream.concat(uriPartitionQualifiedNames.entrySet().stream(), uriTableQualifiedNames.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> {
                        final List<QualifiedName> subNames = Lists.newArrayList(a);
                        subNames.addAll(b);
                        return subNames;
                    }));
                    final List<String> canDeleteMetadataForUris = subUris.parallelStream().filter(s -> !Strings.isNullOrEmpty(s)).filter(s -> uriQualifiedNames.get(s) == null || uriQualifiedNames.get(s).size() == 0).collect(Collectors.toList());
                    log.info("Start deleting data metadata: {}", canDeleteMetadataForUris.size());
                    userMetadataService.deleteDataMetadata(canDeleteMetadataForUris);
                    userMetadataService.deleteDataMetadataDeletes(subUris);
                    MetacatContextManager.removeContext();
                });
            }
            if (urisToDelete.size() < limit) {
                break;
            }
        }
    } catch (Exception e) {
        registry.counter(Metrics.CounterDeleteMetaData.getMetricName()).increment();
        log.warn("Failed deleting data metadata", e);
    }
    log.info("End deleting data metadata");
}