Examples with QualifiedName com.netflix.metacat.common.QualifiedName used on opensource projects

Example 41 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class ElasticSearchRefresh method _processDatabases.

/**
 * Process the list of databases.
 *
 * @param catalogName   catalog name
 * @param databaseNames database names
 * @return future
 */
@SuppressWarnings("checkstyle:methodname")
private ListenableFuture<Void> _processDatabases(final QualifiedName catalogName, final List<QualifiedName> databaseNames) {
    ListenableFuture<Void> resultFuture = null;
    log.info("Full refresh of catalog {} for databases({}): {}", catalogName, databaseNames.size(), databaseNames);
    final List<ListenableFuture<DatabaseDto>> getDatabaseFutures = databaseNames.stream().map(databaseName -> service.submit(() -> {
        DatabaseDto result = null;
        try {
            result = getDatabase(databaseName);
        } catch (Exception e) {
            log.error("Failed to retrieve database: {}", databaseName);
            elasticSearchUtil.log("ElasticSearchRefresh.getDatabase", ElasticSearchDoc.Type.database.name(), databaseName.toString(), null, e.getMessage(), e, true);
        }
        return result;
    })).collect(Collectors.toList());
    if (getDatabaseFutures != null && !getDatabaseFutures.isEmpty()) {
        resultFuture = Futures.transformAsync(Futures.successfulAsList(getDatabaseFutures), input -> {
            final ListenableFuture<Void> processDatabaseFuture = indexDatabaseDtos(catalogName, input);
            final List<ListenableFuture<Void>> processDatabaseFutures = input.stream().filter(NOT_NULL).map(databaseDto -> {
                final List<QualifiedName> tableNames = databaseDto.getTables().stream().map(s -> QualifiedName.ofTable(databaseDto.getName().getCatalogName(), databaseDto.getName().getDatabaseName(), s)).collect(Collectors.toList());
                log.info("Full refresh of database {} for tables({}): {}", databaseDto.getName(), databaseDto.getTables().size(), databaseDto.getTables());
                return processTables(databaseDto.getName(), tableNames);
            }).filter(NOT_NULL).collect(Collectors.toList());
            processDatabaseFutures.add(processDatabaseFuture);
            return Futures.transform(Futures.successfulAsList(processDatabaseFutures), Functions.constant(null));
        });
    }
    return resultFuture;
}

Example 42 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class ElasticSearchRefresh method deleteUnmarkedEntities.

private void deleteUnmarkedEntities(final List<QualifiedName> qNames, final List<QualifiedName> excludeQualifiedNames) {
    log.info("Start: Delete unmarked entities");
    // 
    // get unmarked qualified names
    // check if it not exists
    // delete
    // 
    elasticSearchUtil.refresh();
    final MetacatRequestContext context = MetacatRequestContext.builder().userName("admin").clientAppName("metacat-refresh").apiUri("esRefresh").scheme("internal").build();
    final List<DatabaseDto> unmarkedDatabaseDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("database", qNames, refreshMarker, excludeQualifiedNames, DatabaseDto.class);
    if (!unmarkedDatabaseDtos.isEmpty()) {
        if (unmarkedDatabaseDtos.size() <= config.getElasticSearchThresholdUnmarkedDatabasesDelete()) {
            log.info("Start: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
            final List<String> unmarkedDatabaseNames = Lists.newArrayList();
            final List<DatabaseDto> deleteDatabaseDtos = unmarkedDatabaseDtos.stream().filter(databaseDto -> {
                boolean result = false;
                try {
                    unmarkedDatabaseNames.add(databaseDto.getName().toString());
                    final DatabaseDto dto = databaseService.get(databaseDto.getName(), GetDatabaseServiceParameters.builder().includeUserMetadata(false).includeTableNames(false).disableOnReadMetadataIntercetor(false).build());
                    if (dto == null) {
                        result = true;
                    }
                } catch (DatabaseNotFoundException ignored) {
                    result = true;
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked databases({}): {}", unmarkedDatabaseNames.size(), unmarkedDatabaseNames);
            log.info("Deleting databases({})", deleteDatabaseDtos.size());
            if (!deleteDatabaseDtos.isEmpty()) {
                final List<QualifiedName> deleteDatabaseQualifiedNames = deleteDatabaseDtos.stream().map(DatabaseDto::getName).collect(Collectors.toList());
                final List<String> deleteDatabaseNames = deleteDatabaseQualifiedNames.stream().map(QualifiedName::toString).collect(Collectors.toList());
                log.info("Deleting databases({}): {}", deleteDatabaseNames.size(), deleteDatabaseNames);
                userMetadataService.deleteDefinitionMetadata(deleteDatabaseQualifiedNames);
                elasticSearchUtil.softDelete("database", deleteDatabaseNames, context);
            }
            log.info("End: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
        } else {
            log.info("Count of unmarked databases({}) is more than the threshold {}", unmarkedDatabaseDtos.size(), config.getElasticSearchThresholdUnmarkedDatabasesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedDatabaseThreshholdReached.getMetricName())).increment();
        }
    }
    final List<TableDto> unmarkedTableDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("table", qNames, refreshMarker, excludeQualifiedNames, TableDto.class);
    if (!unmarkedTableDtos.isEmpty()) {
        if (unmarkedTableDtos.size() <= config.getElasticSearchThresholdUnmarkedTablesDelete()) {
            log.info("Start: Delete unmarked tables({})", unmarkedTableDtos.size());
            final List<String> unmarkedTableNames = Lists.newArrayList();
            final List<TableDto> deleteTableDtos = unmarkedTableDtos.stream().filter(tableDto -> {
                boolean result = false;
                try {
                    unmarkedTableNames.add(tableDto.getName().toString());
                    final Optional<TableDto> dto = tableService.get(tableDto.getName(), GetTableServiceParameters.builder().includeDataMetadata(false).disableOnReadMetadataIntercetor(false).includeInfo(true).includeDefinitionMetadata(false).build());
                    if (!dto.isPresent()) {
                        result = true;
                    }
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked tables({}): {}", unmarkedTableNames.size(), unmarkedTableNames);
            log.info("Deleting tables({})", deleteTableDtos.size());
            if (!deleteTableDtos.isEmpty()) {
                final List<String> deleteTableNames = deleteTableDtos.stream().map(dto -> dto.getName().toString()).collect(Collectors.toList());
                log.info("Deleting tables({}): {}", deleteTableNames.size(), deleteTableNames);
                userMetadataService.deleteMetadata("admin", Lists.newArrayList(deleteTableDtos));
                // Publish event. Elasticsearch event handler will take care of updating the index already
                // TODO: Re-evaluate events vs. direct calls for these types of situations like in Genie
                deleteTableDtos.forEach(tableDto -> {
                    tagService.delete(tableDto.getName(), false);
                    this.eventBus.postAsync(new MetacatDeleteTablePostEvent(tableDto.getName(), context, this, tableDto));
                });
            }
            log.info("End: Delete unmarked tables({})", unmarkedTableDtos.size());
        } else {
            log.info("Count of unmarked tables({}) is more than the threshold {}", unmarkedTableDtos.size(), config.getElasticSearchThresholdUnmarkedTablesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedTableThreshholdReached.getMetricName())).increment();
        }
    }
    log.info("End: Delete unmarked entities");
}

Example 43 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MySqlTagService method list.

/**
 * Returns the list of <code>QualifiedName</code> of items that are tagged by the
 * given <code>includeTags</code> and do not contain the given <code>excludeTags</code>.
 *
 * @param includeTags  include items that contain tags
 * @param excludeTags  include items that do not contain tags
 * @param sourceName   catalog/source name
 * @param databaseName database name
 * @param tableName    table name
 * @return list of qualified names of the items
 */
@Override
@Transactional(readOnly = true)
public List<QualifiedName> list(@Nullable final Set<String> includeTags, @Nullable final Set<String> excludeTags, @Nullable final String sourceName, @Nullable final String databaseName, @Nullable final String tableName) {
    Set<String> includedNames = Sets.newHashSet();
    final Set<String> excludedNames = Sets.newHashSet();
    final String wildCardName = QualifiedName.toWildCardString(sourceName, databaseName, tableName);
    // Includes
    final Set<String> localIncludes = includeTags != null ? includeTags : Sets.newHashSet();
    try {
        String query = String.format(QUERY_SEARCH, "in ('" + Joiner.on("','").skipNulls().join(localIncludes) + "')");
        final Object[] params = { localIncludes.size() == 0 ? 1 : 0, wildCardName == null ? 1 : 0, wildCardName };
        includedNames.addAll(jdbcTemplate.query(query, params, new int[] { Types.INTEGER, Types.INTEGER, Types.VARCHAR }, (rs, rowNum) -> rs.getString("name")));
        if (excludeTags != null && !excludeTags.isEmpty()) {
            // Excludes
            query = String.format(QUERY_SEARCH, "in ('" + Joiner.on("','").skipNulls().join(excludeTags) + "')");
            final Object[] eParams = { excludeTags.size() == 0 ? 1 : 0, wildCardName == null ? 1 : 0, wildCardName };
            excludedNames.addAll(jdbcTemplate.query(query, eParams, new int[] { Types.INTEGER, Types.INTEGER, Types.VARCHAR }, (rs, rowNum) -> rs.getString("name")));
        }
    } catch (Exception e) {
        final String message = String.format("Failed getting the list of qualified names for tags %s", includeTags);
        log.error(message, e);
        throw new UserMetadataServiceException(message, e);
    }
    if (excludeTags != null && !excludeTags.isEmpty()) {
        includedNames = Sets.difference(includedNames, excludedNames);
    }
    return includedNames.stream().map(s -> QualifiedName.fromString(s, false)).collect(Collectors.toList());
}

Example 44 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class CatalogThriftHiveMetastore method alter_table_with_environment_context.

/**
 * {@inheritDoc}
 */
@Override
public void alter_table_with_environment_context(final String dbname, final String tblName, final Table newTbl, @Nullable final EnvironmentContext environmentContext) throws TException {
    requestWrapper("alter_table_with_environment_context", new Object[] { dbname, tblName, newTbl, environmentContext }, () -> {
        final String databaseName = normalizeIdentifier(dbname);
        final String tableName = normalizeIdentifier(tblName);
        final QualifiedName oldName = QualifiedName.ofTable(catalogName, databaseName, tableName);
        final QualifiedName newName = QualifiedName.ofTable(catalogName, newTbl.getDbName(), newTbl.getTableName());
        final TableDto dto = hiveConverters.hiveToMetacatTable(newName, newTbl);
        if (!oldName.equals(newName)) {
            v1.renameTable(catalogName, oldName.getDatabaseName(), oldName.getTableName(), newName.getTableName());
        }
        v1.updateTable(catalogName, dbname, newName.getTableName(), dto);
        return null;
    });
}

Example 45 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class CatalogThriftHiveMetastore method create_table_with_environment_context.

/**
 * {@inheritDoc}
 */
@Override
public void create_table_with_environment_context(final Table tbl, @Nullable final EnvironmentContext environmentContext) throws TException {
    requestWrapper("create_table_with_environment_context", new Object[] { tbl, environmentContext }, () -> {
        final String dbname = normalizeIdentifier(tbl.getDbName());
        final String tblName = normalizeIdentifier(tbl.getTableName());
        final QualifiedName name = QualifiedName.ofTable(catalogName, dbname, tblName);
        final TableDto dto = hiveConverters.hiveToMetacatTable(name, tbl);
        v1.createTable(catalogName, dbname, tblName, dto);
        return null;
    });
}