Examples with PartitionDto com.netflix.metacat.common.dto.PartitionDto used on opensource projects

Example 11 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class PartitionServiceImpl method list.

@Override
public List<PartitionDto> list(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, final boolean includePartitionDetails) {
    if (Strings.isNullOrEmpty(filter) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getQualifiedNamesToThrowErrorWhenNoFilterOnListPartitions().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
    requestDto.setFilter(filter);
    requestDto.setIncludePartitionDetails(includePartitionDetails);
    requestDto.setPartitionNames(partitionNames);
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.gauge(this.partitionGetCountId.withTags(new HashMap<>(name.parts())), result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filter, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}

Example 12 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class PartitionServiceImpl method get.

@Override
public PartitionDto get(@Nonnull final QualifiedName name) {
    PartitionDto result = null;
    final QualifiedName tableName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), name.getTableName());
    final List<PartitionDto> dtos = list(tableName, null, Lists.newArrayList(name.getPartitionName()), null, null, true, true, true);
    if (!dtos.isEmpty()) {
        result = dtos.get(0);
    }
    return result;
}

Example 13 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class MetacatServiceHelper method postPostUpdateEvent.

/**
     * Calls the right method of the event bus for the given qualified name.
     *
     * @param name                  name
     * @param metacatRequestContext context
     * @param oldDTo                dto
     * @param currentDto            dto
     */
public void postPostUpdateEvent(final QualifiedName name, final MetacatRequestContext metacatRequestContext, final BaseDto oldDTo, final BaseDto currentDto) {
    if (name.isPartitionDefinition()) {
        final List<PartitionDto> dtos = Lists.newArrayList();
        if (currentDto != null) {
            dtos.add((PartitionDto) currentDto);
        }
        // This request neither added nor updated partitions
        final PartitionsSaveResponseDto partitionsSaveResponseDto = new PartitionsSaveResponseDto();
        this.eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, dtos, partitionsSaveResponseDto));
    } else if (name.isTableDefinition()) {
        final MetacatUpdateTablePostEvent event = new MetacatUpdateTablePostEvent(name, metacatRequestContext, this, (TableDto) oldDTo, (TableDto) currentDto);
        this.eventBus.postAsync(event);
    } else if (name.isDatabaseDefinition()) {
        this.eventBus.postAsync(new MetacatUpdateDatabasePostEvent(name, metacatRequestContext, this));
    } else {
        throw new IllegalArgumentException(String.format("Invalid name %s", name));
    }
}

Example 14 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class MetacatElasticSearchEventHandlers method metacatSaveTablePartitionPostEventHandler.

/**
     * Subscriber.
     *
     * @param event event
     */
@EventListener
public void metacatSaveTablePartitionPostEventHandler(final MetacatSaveTablePartitionPostEvent event) {
    log.debug("Received SaveTablePartitionEvent {}", event);
    registry.counter(Metrics.CounterElasticSearchPartitionSave.name()).increment();
    final List<PartitionDto> partitionDtos = event.getPartitions();
    final MetacatRequestContext context = event.getRequestContext();
    final List<ElasticSearchDoc> docs = partitionDtos.stream().map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, context.getUserName(), false)).collect(Collectors.toList());
    es.save(ElasticSearchDoc.Type.partition.name(), docs);
}

Example 15 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class CatalogThriftHiveMetastore method getPartitionDtoByName.

private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException {
    final GetPartitionsRequestDto dto = new GetPartitionsRequestDto();
    dto.setIncludePartitionDetails(true);
    dto.setPartitionNames(ImmutableList.of(partName));
    final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest(catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto);
    if (partitionDtos == null || partitionDtos.isEmpty()) {
        throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName());
    } else if (partitionDtos.size() != 1) {
        // I don't think this is even possible
        throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName());
    }
    return partitionDtos.get(0);
}