Examples with PartitionListRequest com.netflix.metacat.common.server.connectors.model.PartitionListRequest used on opensource projects

Example 1 with PartitionListRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionListRequest in project metacat by Netflix.

the class DirectSqlGetPartition method getPartitionKeys.

/**
 * Gets the partition names/keys based on a filter expression for the specified table.
 *
 * @param requestContext    The Metacat request context
 * @param tableName         table handle to get partition for
 * @param partitionsRequest The metadata for what kind of partitions to get from the table
 * @return filtered list of partition names
 */
@Transactional(readOnly = true)
public List<String> getPartitionKeys(final ConnectorRequestContext requestContext, final QualifiedName tableName, final PartitionListRequest partitionsRequest) {
    final long start = registry.clock().wallTime();
    final List<String> result;
    final List<String> partitionNames = partitionsRequest.getPartitionNames();
    final Sort sort = partitionsRequest.getSort();
    final Pageable pageable = partitionsRequest.getPageable();
    final String filterExpression = partitionsRequest.getFilter();
    if (filterExpression != null) {
        return filterPartitionsColumn(tableName.getDatabaseName(), tableName.getTableName(), partitionNames, PARTITION_NAME, filterExpression, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    } else {
        final ResultSetExtractor<List<String>> handler = rs -> {
            final List<String> names = Lists.newArrayList();
            while (rs.next()) {
                names.add(rs.getString("name"));
            }
            return names;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), null, partitionNames, SQL.SQL_GET_PARTITIONS_WITH_KEY, handler, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    }
    this.fastServiceMetric.recordTimer(HiveMetrics.TagGetPartitionKeys.getMetricName(), registry.clock().wallTime() - start);
    return result;
}

Example 2 with PartitionListRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionListRequest in project metacat by Netflix.

the class DirectSqlGetPartition method getPartitionUris.

/**
 * Gets the partition uris based on a filter expression for the specified table.
 *
 * @param requestContext    The Metacat request context
 * @param tableName         table handle to get partition for
 * @param partitionsRequest The metadata for what kind of partitions to get from the table
 * @return filtered list of partition names
 */
@Transactional(readOnly = true)
public List<String> getPartitionUris(final ConnectorRequestContext requestContext, final QualifiedName tableName, final PartitionListRequest partitionsRequest) {
    final long start = registry.clock().wallTime();
    final List<String> result;
    final List<String> partitionNames = partitionsRequest.getPartitionNames();
    final Sort sort = partitionsRequest.getSort();
    final Pageable pageable = partitionsRequest.getPageable();
    final String filterExpression = partitionsRequest.getFilter();
    if (filterExpression != null) {
        return filterPartitionsColumn(tableName.getDatabaseName(), tableName.getTableName(), partitionNames, PARTITION_URI, filterExpression, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    } else {
        final ResultSetExtractor<List<String>> handler = rs -> {
            final List<String> uris = Lists.newArrayList();
            while (rs.next()) {
                uris.add(rs.getString(PARTITION_URI));
            }
            return uris;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), null, partitionNames, SQL.SQL_GET_PARTITIONS_URI, handler, sort, pageable, partitionsRequest.getIncludeAuditOnly());
    }
    this.fastServiceMetric.recordTimer(HiveMetrics.TagGetPartitionKeys.getMetricName(), registry.clock().wallTime() - start);
    return result;
}

Example 3 with PartitionListRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionListRequest in project metacat by Netflix.

the class HiveConnectorFastPartitionService method getPartitionKeys.

/**
     * {@inheritDoc}.
     */
@Override
public List<String> getPartitionKeys(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName, @Nonnull @NonNull final PartitionListRequest partitionsRequest) {
    final long start = registry.clock().monotonicTime();
    final Map<String, String> tags = new HashMap<String, String>();
    tags.put("request", "getPartitionKeys");
    final List<String> result;
    final List<String> partitionNames = partitionsRequest.getPartitionNames();
    final Sort sort = partitionsRequest.getSort();
    final Pageable pageable = partitionsRequest.getPageable();
    final String filterExpression = partitionsRequest.getFilter();
    if (filterExpression != null) {
        final FilterPartition filter = new FilterPartition();
        // batch exists
        final boolean isBatched = !Strings.isNullOrEmpty(filterExpression) && filterExpression.contains(FIELD_BATCHID);
        final boolean hasDateCreated = !Strings.isNullOrEmpty(filterExpression) && filterExpression.contains(FIELD_DATE_CREATED);
        // Handler for reading the result set
        final ResultSetHandler<List<String>> handler = rs -> {
            final List<String> names = Lists.newArrayList();
            while (rs.next()) {
                final String name = rs.getString("name");
                final String uri = rs.getString("uri");
                final long createdDate = rs.getLong(FIELD_DATE_CREATED);
                Map<String, String> values = null;
                if (hasDateCreated) {
                    values = Maps.newHashMap();
                    values.put(FIELD_DATE_CREATED, createdDate + "");
                }
                if (Strings.isNullOrEmpty(filterExpression) || filter.evaluatePartitionExpression(filterExpression, name, uri, isBatched, values)) {
                    names.add(name);
                }
            }
            return names;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), filterExpression, partitionNames, SQL_GET_PARTITIONS_WITH_KEY_URI, handler, sort, pageable);
    } else {
        // Handler for reading the result set
        final ResultSetHandler<List<String>> handler = rs -> {
            final List<String> names = Lists.newArrayList();
            while (rs.next()) {
                names.add(rs.getString("name"));
            }
            return names;
        };
        result = getHandlerResults(tableName.getDatabaseName(), tableName.getTableName(), null, partitionNames, SQL_GET_PARTITIONS_WITH_KEY, handler, sort, pageable);
    }
    final long duration = registry.clock().monotonicTime() - start;
    log.debug("### Time taken to complete getPartitionKeys is {} ms", duration);
    this.registry.timer(requestTimerId.withTags(tags)).record(duration, TimeUnit.MILLISECONDS);
    return result;
}

Example 4 with PartitionListRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionListRequest in project metacat by Netflix.

the class ConverterUtil method toPartitionListRequest.

/**
 * Creates the partition list connector request.
 *
 * @param partitionsRequestDto request containing the filter and other properties used for listing
 * @param pageable             pageable info
 * @param sort                 sort info
 * @return connector request
 */
public PartitionListRequest toPartitionListRequest(final GetPartitionsRequestDto partitionsRequestDto, final Pageable pageable, final Sort sort) {
    if (partitionsRequestDto != null) {
        if (partitionsRequestDto.getIncludePartitionDetails() == null) {
            partitionsRequestDto.setIncludePartitionDetails(false);
        }
        if (partitionsRequestDto.getIncludeAuditOnly() == null) {
            partitionsRequestDto.setIncludeAuditOnly(false);
        }
        final PartitionListRequest result = mapper.map(partitionsRequestDto, PartitionListRequest.class);
        result.setPageable(pageable);
        result.setSort(sort);
        return result;
    } else {
        return new PartitionListRequest(null, null, false, pageable, sort, false);
    }
}

Example 5 with PartitionListRequest

use of com.netflix.metacat.common.server.connectors.model.PartitionListRequest in project metacat by Netflix.

the class PartitionServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionDto> list(final QualifiedName name, @Nullable final Sort sort, @Nullable final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, @Nullable final GetPartitionsRequestDto getPartitionsRequestDto) {
    // add table info here
    // the conversion will handle getPartitionsRequestDto as null case
    final PartitionListRequest partitionListRequest = converterUtil.toPartitionListRequest(getPartitionsRequestDto, pageable, sort);
    final String filterExpression = partitionListRequest.getFilter();
    final List<String> partitionNames = partitionListRequest.getPartitionNames();
    if (Strings.isNullOrEmpty(filterExpression) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getNamesToThrowErrorOnListPartitionsWithNoFilter().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name);
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorRequestContext, name, partitionListRequest, this.getTableInfo(name));
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        final Map<String, ObjectNode> prePopulatedMap = new HashMap<>();
        resultInfo.stream().filter(partitionInfo -> partitionInfo.getDataMetrics() != null).forEach(partitionInfo -> prePopulatedMap.put(partitionInfo.getName().toString(), partitionInfo.getDataMetrics()));
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            if (partitionDto.isDataExternal()) {
                uris.add(partitionDto.getDataUri());
            }
        });
        registry.distributionSummary(this.partitionGetDistSummary.withTags(name.parts())).record(result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filterExpression, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), prePopulatedMap.containsKey(partitionDto.getName().toString()) ? // using the prepopulated datametric
                prePopulatedMap.get(partitionDto.getName().toString()) : dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}