Examples with QualifiedName com.netflix.metacat.common.QualifiedName used on opensource projects

Example 36 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class DirectSqlGetPartition method getPartitionUris.

/**
 * Gets the partition uris based on a filter expression for the specified table.
 *
 * @param requestContext    The Metacat request context
 * @param tableName         table handle to get partition for
 * @param partitionsRequest The metadata for what kind of partitions to get from the table
 * @return filtered list of partition names
 */
@Transactional(readOnly = true)
public List<String> getPartitionUris(final ConnectorRequestContext requestContext, final QualifiedName tableName, final PartitionListRequest partitionsRequest) {
    final long start = registry.clock().wallTime();
    final List<String> result;
    final List<String> partitionNames = partitionsRequest.getPartitionNames();
    final Sort sort = partitionsRequest.getSort();
    final Pageable pageable = partitionsRequest.getPageable();
    final String filterExpression = partitionsRequest.getFilter();
    if (filterExpression != null) {
        return filterPartitionsColumn(tableName.getDatabaseName(), tableName.getTableName(), partitionNames, PARTITION_URI, filterExpression, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    } else {
        final ResultSetExtractor<List<String>> handler = rs -> {
            final List<String> uris = Lists.newArrayList();
            while (rs.next()) {
                uris.add(rs.getString(PARTITION_URI));
            }
            return uris;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), null, partitionNames, SQL.SQL_GET_PARTITIONS_URI, handler, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    }
    this.fastServiceMetric.recordTimer(HiveMetrics.TagGetPartitionKeys.getMetricName(), registry.clock().wallTime() - start);
    return result;
}

Example 37 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class DirectSqlTable method getTableNames.

/**
 * Returns all the table names referring to the given <code>uris</code>.
 *
 * @param uris         locations
 * @param prefixSearch if true, we look for tables whose location starts with the given <code>uri</code>
 * @return map of uri to list of partition names
 * @throws UnsupportedOperationException If the connector doesn't implement this method
 */
@Transactional(readOnly = true)
public Map<String, List<QualifiedName>> getTableNames(final List<String> uris, final boolean prefixSearch) {
    final long start = registry.clock().wallTime();
    // Create the sql
    final StringBuilder queryBuilder = new StringBuilder(SQL.GET_TABLE_NAMES_BY_URI);
    final List<SqlParameterValue> params = Lists.newArrayList();
    if (prefixSearch) {
        queryBuilder.append(" and (1=0");
        uris.forEach(uri -> {
            queryBuilder.append(" or location like ?");
            params.add(new SqlParameterValue(Types.VARCHAR, uri + "%"));
        });
        queryBuilder.append(" )");
    } else {
        queryBuilder.append(" and location in (");
        uris.forEach(uri -> {
            queryBuilder.append("?,");
            params.add(new SqlParameterValue(Types.VARCHAR, uri));
        });
        queryBuilder.deleteCharAt(queryBuilder.length() - 1).append(")");
    }
    ResultSetExtractor<Map<String, List<QualifiedName>>> handler = rs -> {
        final Map<String, List<QualifiedName>> result = Maps.newHashMap();
        while (rs.next()) {
            final String schemaName = rs.getString("schema_name");
            final String tableName = rs.getString("table_name");
            final String uri = rs.getString("location");
            final List<QualifiedName> names = result.computeIfAbsent(uri, k -> Lists.newArrayList());
            names.add(QualifiedName.ofTable(catalogName, schemaName, tableName));
        }
        return result;
    };
    try {
        return jdbcTemplate.query(queryBuilder.toString(), params.toArray(), handler);
    } finally {
        this.fastServiceMetric.recordTimer(HiveMetrics.TagGetTableNames.getMetricName(), registry.clock().wallTime() - start);
    }
}

Example 38 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class HiveConnectorFastTableService method update.

/**
 * Update a table with the given metadata.
 *
 * If table is an iceberg table, then lock the table for update so that no other request can update it. If the meta
 * information is invalid, then throw an error.
 * If table is not an iceberg table, then do a regular table update.
 *
 * @param requestContext The request context
 * @param tableInfo      The resource metadata
 */
@Override
public void update(final ConnectorRequestContext requestContext, final TableInfo tableInfo) {
    if (isIcebergTable(tableInfo)) {
        final QualifiedName tableName = tableInfo.getName();
        final Long tableId = directSqlTable.getTableId(tableName);
        try {
            log.debug("Locking Iceberg table {}", tableName);
            directSqlTable.lockIcebergTable(tableId, tableName);
            try {
                final TableInfo existingTableInfo = get(requestContext, tableInfo.getName());
                validateIcebergUpdate(existingTableInfo, tableInfo);
                final Table existingTable = getHiveMetacatConverters().fromTableInfo(existingTableInfo);
                super.update(requestContext, existingTable, tableInfo);
            } finally {
                directSqlTable.unlockIcebergTable(tableId);
                log.debug("Unlocked Iceberg table {}", tableName);
            }
        } catch (IllegalStateException e) {
            throw new TablePreconditionFailedException(tableName, e.getMessage());
        }
    } else {
        super.update(requestContext, tableInfo);
    }
}

Example 39 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class ElasticSearchRefresh method _processCatalogs.

@SuppressWarnings("checkstyle:methodname")
private ListenableFuture<Void> _processCatalogs(final List<String> catalogNames) {
    log.info("Start: Full refresh of catalogs: {}", catalogNames);
    final List<ListenableFuture<CatalogDto>> getCatalogFutures = catalogNames.stream().map(catalogName -> service.submit(() -> {
        CatalogDto result = null;
        try {
            result = getCatalog(catalogName);
        } catch (Exception e) {
            log.error("Failed to retrieve catalog: {}", catalogName);
            elasticSearchUtil.log("ElasticSearchRefresh.getCatalog", ElasticSearchDoc.Type.catalog.name(), catalogName, null, e.getMessage(), e, true);
        }
        return result;
    })).collect(Collectors.toList());
    return Futures.transformAsync(Futures.successfulAsList(getCatalogFutures), input -> {
        final List<ListenableFuture<Void>> processCatalogFutures = input.stream().filter(NOT_NULL).map(catalogDto -> {
            final List<QualifiedName> databaseNames = getDatabaseNamesToRefresh(catalogDto);
            return _processDatabases(catalogDto.getName(), databaseNames);
        }).filter(NOT_NULL).collect(Collectors.toList());
        return Futures.transform(Futures.successfulAsList(processCatalogFutures), Functions.constant(null));
    });
}

Example 40 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class ElasticSearchRefresh method _processPartitions.

@SuppressWarnings("checkstyle:methodname")
private ListenableFuture<Void> _processPartitions(final List<QualifiedName> qNames) {
    final List<QualifiedName> excludeQualifiedNames = config.getElasticSearchRefreshExcludeQualifiedNames();
    final List<String> tables = elasticSearchUtil.getTableIdsByCatalogs(ElasticSearchDoc.Type.table.name(), qNames, excludeQualifiedNames);
    final List<ListenableFuture<ListenableFuture<Void>>> futures = tables.stream().map(s -> service.submit(() -> {
        final QualifiedName tableName = QualifiedName.fromString(s, false);
        final List<ListenableFuture<Void>> indexFutures = Lists.newArrayList();
        int offset = 0;
        int count;
        final Sort sort;
        if ("s3".equals(tableName.getCatalogName()) || "aegisthus".equals(tableName.getCatalogName())) {
            sort = new Sort("id", SortOrder.ASC);
        } else {
            sort = new Sort("part_id", SortOrder.ASC);
        }
        final Pageable pageable = new Pageable(10000, offset);
        do {
            final List<PartitionDto> partitionDtos = partitionService.list(tableName, sort, pageable, true, true, new GetPartitionsRequestDto(null, null, true, true));
            count = partitionDtos.size();
            if (!partitionDtos.isEmpty()) {
                final List<List<PartitionDto>> partitionedPartitionDtos = Lists.partition(partitionDtos, 1000);
                partitionedPartitionDtos.forEach(subPartitionsDtos -> indexFutures.add(indexPartitionDtos(tableName, subPartitionsDtos)));
                offset = offset + count;
                pageable.setOffset(offset);
            }
        } while (count == 10000);
        return Futures.transform(Futures.successfulAsList(indexFutures), Functions.constant((Void) null));
    })).collect(Collectors.toList());
    final ListenableFuture<Void> processPartitionsFuture = Futures.transformAsync(Futures.successfulAsList(futures), input -> {
        final List<ListenableFuture<Void>> inputFuturesWithoutNulls = input.stream().filter(NOT_NULL).collect(Collectors.toList());
        return Futures.transform(Futures.successfulAsList(inputFuturesWithoutNulls), Functions.constant(null));
    });
    return Futures.transformAsync(processPartitionsFuture, input -> {
        elasticSearchUtil.refresh();
        final List<ListenableFuture<Void>> cleanUpFutures = tables.stream().map(s -> service.submit(() -> partitionsCleanUp(QualifiedName.fromString(s, false), excludeQualifiedNames))).collect(Collectors.toList());
        return Futures.transform(Futures.successfulAsList(cleanUpFutures), Functions.constant(null));
    });
}