Examples with ConnectorPartitionService com.netflix.metacat.common.server.connectors.ConnectorPartitionService used on opensource projects

Example 6 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method getQualifiedNames.

@Override
public Map<String, List<QualifiedName>> getQualifiedNames(final List<String> uris, final boolean prefixSearch) {
    final Map<String, List<QualifiedName>> result = Maps.newConcurrentMap();
    final List<ListenableFuture<Void>> futures = Lists.newArrayList();
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    catalogService.getCatalogNames().forEach(catalog -> {
        futures.add(threadServiceManager.getExecutor().submit(() -> {
            final ConnectorPartitionService service = connectorManager.getPartitionService(catalog.getCatalogName());
            final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
            try {
                final Map<String, List<QualifiedName>> partitionNames = service.getPartitionNames(connectorContext, uris, prefixSearch);
                partitionNames.forEach((uri, subPartitionNames) -> {
                    final List<QualifiedName> existingPartitionNames = result.get(uri);
                    if (existingPartitionNames == null) {
                        result.put(uri, subPartitionNames);
                    } else {
                        existingPartitionNames.addAll(subPartitionNames);
                    }
                });
            } catch (final UnsupportedOperationException uoe) {
                log.debug("Catalog {} doesn't support getPartitionNames. Ignoring.", catalog.getCatalogName());
            }
            return null;
        }));
    });
    try {
        Futures.allAsList(futures).get(1, TimeUnit.HOURS);
    } catch (Exception e) {
        Throwables.propagate(e);
    }
    return result;
}

Example 7 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method list.

@Override
public List<PartitionDto> list(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, final boolean includePartitionDetails) {
    if (Strings.isNullOrEmpty(filter) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getQualifiedNamesToThrowErrorWhenNoFilterOnListPartitions().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
    requestDto.setFilter(filter);
    requestDto.setIncludePartitionDetails(includePartitionDetails);
    requestDto.setPartitionNames(partitionNames);
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.gauge(this.partitionGetCountId.withTags(new HashMap<>(name.parts())), result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filter, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}