Examples with QualifiedName com.netflix.metacat.common.QualifiedName used on opensource projects

Example 51 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MViewServiceEventHandler method metacatRenameTablePostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatRenameTablePostEventHandler(final MetacatRenameTablePostEvent event) {
    if (!event.isMView()) {
        final QualifiedName oldName = event.getOldTable().getName();
        final QualifiedName newName = event.getCurrentTable().getName();
        final List<NameDateDto> views = mViewService.list(oldName);
        if (views != null && !views.isEmpty()) {
            views.forEach(view -> {
                final QualifiedName newViewName = QualifiedName.ofView(oldName.getCatalogName(), oldName.getDatabaseName(), newName.getTableName(), view.getName().getViewName());
                mViewService.rename(view.getName(), newViewName);
            });
        }
    }
}

Example 52 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MViewServiceEventHandler method metacatDeleteTablePostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatDeleteTablePostEventHandler(final MetacatDeleteTablePostEvent event) {
    if (config.canCascadeViewsMetadataOnTableDelete() && !event.isMView()) {
        final QualifiedName name = event.getTable().getName();
        try {
            // delete views associated with this table
            final List<NameDateDto> viewNames = mViewService.list(name);
            viewNames.forEach(viewName -> mViewService.deleteAndReturn(viewName.getName()));
        } catch (Exception e) {
            log.warn("Failed cleaning mviews after deleting table {}", name);
        }
        // delete table partitions metadata
        try {
            final List<QualifiedName> names = userMetadataService.getDescendantDefinitionNames(name);
            if (names != null && !names.isEmpty()) {
                userMetadataService.deleteDefinitionMetadata(names);
            }
        } catch (Exception e) {
            log.warn("Failed cleaning partition definition metadata after deleting table {}", name);
        }
    }
}

Example 53 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MetadataService method cleanUpDeletedDataMetadata.

/**
 * Deletes all the data metadata marked for deletion.
 */
public void cleanUpDeletedDataMetadata() {
    // Get the data metadata that were marked deleted a number of days back
    // Check if the uri is being used
    // If uri is not used then delete the entry from data_metadata
    log.info("Start deleting data metadata");
    try {
        final DateTime priorTo = DateTime.now().minusDays(config.getDataMetadataDeleteMarkerLifetimeInDays());
        final int limit = 100000;
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        while (true) {
            final List<String> urisToDelete = userMetadataService.getDeletedDataMetadataUris(priorTo.toDate(), 0, limit);
            log.info("Count of deleted marked data metadata: {}", urisToDelete.size());
            if (urisToDelete.size() > 0) {
                final List<String> uris = urisToDelete.parallelStream().filter(uri -> !uri.contains("=")).map(userMetadataService::getDescendantDataUris).flatMap(Collection::stream).collect(Collectors.toList());
                uris.addAll(urisToDelete);
                log.info("Count of deleted marked data metadata (including descendants) : {}", uris.size());
                final List<List<String>> subListsUris = Lists.partition(uris, 1000);
                subListsUris.parallelStream().forEach(subUris -> {
                    MetacatContextManager.setContext(metacatRequestContext);
                    final Map<String, List<QualifiedName>> uriPartitionQualifiedNames = partitionService.getQualifiedNames(subUris, false);
                    final Map<String, List<QualifiedName>> uriTableQualifiedNames = tableService.getQualifiedNames(subUris, false);
                    final Map<String, List<QualifiedName>> uriQualifiedNames = Stream.concat(uriPartitionQualifiedNames.entrySet().stream(), uriTableQualifiedNames.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> {
                        final List<QualifiedName> subNames = Lists.newArrayList(a);
                        subNames.addAll(b);
                        return subNames;
                    }));
                    final List<String> canDeleteMetadataForUris = subUris.parallelStream().filter(s -> !Strings.isNullOrEmpty(s)).filter(s -> uriQualifiedNames.get(s) == null || uriQualifiedNames.get(s).size() == 0).collect(Collectors.toList());
                    log.info("Start deleting data metadata: {}", canDeleteMetadataForUris.size());
                    userMetadataService.deleteDataMetadata(canDeleteMetadataForUris);
                    userMetadataService.deleteDataMetadataDeletes(subUris);
                    MetacatContextManager.removeContext();
                });
            }
            if (urisToDelete.size() < limit) {
                break;
            }
        }
    } catch (Exception e) {
        registry.counter(Metrics.CounterDeleteMetaData.getMetricName()).increment();
        log.warn("Failed deleting data metadata", e);
    }
    log.info("End deleting data metadata");
}

Example 54 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class CatalogServiceImpl method get.

/**
 * {@inheritDoc}
 */
@Nonnull
@Override
public CatalogDto get(final QualifiedName name) {
    final Set<MetacatCatalogConfig> configs = connectorManager.getCatalogConfigs(name.getCatalogName());
    final CatalogDto result = new CatalogDto();
    result.setName(name);
    final ConnectorRequestContext context = converterUtil.toConnectorContext(MetacatContextManager.getContext());
    final List<String> databases = Lists.newArrayList();
    configs.forEach(config -> {
        QualifiedName qName = name;
        if (config.getSchemaWhitelist().isEmpty()) {
            result.setType(config.getType());
        } else {
            qName = QualifiedName.ofDatabase(name.getCatalogName(), config.getSchemaWhitelist().get(0));
        }
        databases.addAll(connectorManager.getDatabaseService(qName).listNames(context, name, null, null, null).stream().map(QualifiedName::getDatabaseName).filter(s -> config.getSchemaBlacklist().isEmpty() || !config.getSchemaBlacklist().contains(s)).filter(s -> config.getSchemaWhitelist().isEmpty() || config.getSchemaWhitelist().contains(s)).sorted(String.CASE_INSENSITIVE_ORDER).collect(Collectors.toList()));
    });
    result.setDatabases(databases);
    userMetadataService.populateMetadata(result, false);
    return result;
}

Example 55 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class DatabaseServiceImpl method get.

/**
 * {@inheritDoc}
 */
@Override
public DatabaseDto get(final QualifiedName name, final GetDatabaseServiceParameters getDatabaseServiceParameters) {
    validate(name);
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final MetacatCatalogConfig config = connectorManager.getCatalogConfig(name);
    final ConnectorDatabaseService service = connectorManager.getDatabaseService(name);
    final ConnectorTableService tableService = connectorManager.getTableService(name);
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    final DatabaseDto dto = converterUtil.toDatabaseDto(service.get(connectorRequestContext, name));
    dto.setType(config.getType());
    if (getDatabaseServiceParameters.isIncludeTableNames()) {
        final List<QualifiedName> tableNames = tableService.listNames(connectorRequestContext, name, null, null, null);
        List<QualifiedName> viewNames = Collections.emptyList();
        if (config.isIncludeViewsWithTables()) {
            // TODO JdbcMetadata returns ImmutableList.of() for views.  We should change it to fetch views.
            try {
                viewNames = service.listViewNames(connectorRequestContext, name);
            } catch (UnsupportedOperationException ignored) {
            }
        }
        // Check to see if schema exists
        if (tableNames.isEmpty() && viewNames.isEmpty() && !exists(name)) {
            throw new DatabaseNotFoundException(name);
        }
        dto.setTables(Stream.concat(tableNames.stream(), viewNames.stream()).map(QualifiedName::getTableName).sorted(String.CASE_INSENSITIVE_ORDER).collect(Collectors.toList()));
    }
    if (getDatabaseServiceParameters.isIncludeUserMetadata()) {
        log.info("Populate user metadata for schema {}", name);
        userMetadataService.populateMetadata(dto, getDatabaseServiceParameters.isDisableOnReadMetadataIntercetor());
    }
    return dto;
}