Examples with GetPartitionsRequestDto com.netflix.metacat.common.dto.GetPartitionsRequestDto used on opensource projects

Example 1 with GetPartitionsRequestDto

use of com.netflix.metacat.common.dto.GetPartitionsRequestDto in project metacat by Netflix.

the class CatalogThriftHiveMetastore method get_partitions_by_names.

/**
     * {@inheritDoc}
     */
@Override
public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> names) throws TException {
    return requestWrapper("get_partitions_by_names", new Object[] { dbName, tblName, names }, () -> {
        final String databaseName = normalizeIdentifier(dbName);
        final String tableName = normalizeIdentifier(tblName);
        final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false);
        final GetPartitionsRequestDto dto = new GetPartitionsRequestDto();
        dto.setIncludePartitionDetails(true);
        dto.setPartitionNames(names);
        final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, databaseName, tableName, null, null, null, null, false, dto);
        final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size());
        for (PartitionDto partition : metacatPartitions) {
            result.add(hiveConverters.metacatToHivePartition(partition, tableDto));
        }
        return result;
    });
}

Example 2 with GetPartitionsRequestDto

use of com.netflix.metacat.common.dto.GetPartitionsRequestDto in project metacat by Netflix.

the class PartitionServiceImpl method getPartitionKeys.

@Override
public List<String> getPartitionKeys(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable) {
    List<String> result = Lists.newArrayList();
    if (tableService.exists(name)) {
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setFilter(filter);
        requestDto.setPartitionNames(partitionNames);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        try {
            result = service.getPartitionKeys(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
        } catch (final UnsupportedOperationException uoe) {
            log.debug("Catalog {} doesn't support getPartitionKeys. Ignoring.", name.getCatalogName());
        }
    }
    return result;
}

Example 3 with GetPartitionsRequestDto

use of com.netflix.metacat.common.dto.GetPartitionsRequestDto in project metacat by Netflix.

the class PartitionServiceImpl method save.

@Override
public PartitionsSaveResponseDto save(@Nonnull final QualifiedName name, final PartitionsSaveRequestDto dto) {
    PartitionsSaveResponseDto result = new PartitionsSaveResponseDto();
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final List<PartitionDto> partitionDtos = dto.getPartitions();
    // If no partitions are passed, then return
    if (partitionDtos == null || partitionDtos.isEmpty()) {
        return result;
    }
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    registry.gauge(this.partitionAddedCountId.withTags(new HashMap<>(name.parts())), partitionDtos.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.gauge(this.partitionDeletedCountId.withTags(new HashMap<>(name.parts())), partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setIncludePartitionDetails(false);
        requestDto.setPartitionNames(partitionIdsForDeletes);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        if (deletePartitionInfos != null) {
            deletePartitions = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        }
    }
    //
    // Save all the new and updated partitions
    //
    eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions({}) for {}", partitionDtos.size(), name);
    result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    userMetadataService.saveMetadatas(metacatRequestContext.getUserName(), partitionDtos, true);
    eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionIdsForDeletes));
    }
    return result;
}

Example 4 with GetPartitionsRequestDto

use of com.netflix.metacat.common.dto.GetPartitionsRequestDto in project metacat by Netflix.

the class PartitionServiceImpl method getPartitionUris.

@Override
public List<String> getPartitionUris(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable) {
    List<String> result = Lists.newArrayList();
    if (tableService.exists(name)) {
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setFilter(filter);
        requestDto.setPartitionNames(partitionNames);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        try {
            result = service.getPartitionUris(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
        } catch (final UnsupportedOperationException uoe) {
            log.info("Catalog {} doesn't support getPartitionUris. Ignoring.", name.getCatalogName());
        }
    }
    return result;
}

Example 5 with GetPartitionsRequestDto

use of com.netflix.metacat.common.dto.GetPartitionsRequestDto in project metacat by Netflix.

the class PartitionServiceImpl method delete.

@Override
public void delete(final QualifiedName name, final List<String> partitionIds) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    registry.gauge(this.partitionDeletedCountId.withTags(new HashMap<>(name.parts())), partitionIds.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    if (!partitionIds.isEmpty()) {
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitionIdsForDeletes(partitionIds);
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        // Get the partitions before calling delete
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setIncludePartitionDetails(false);
        requestDto.setPartitionNames(partitionIds);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        final List<PartitionInfo> partitionInfos = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        List<HasMetadata> partitions = Lists.newArrayList();
        if (partitionInfos != null) {
            partitions = partitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        }
        log.info("Deleting partitions with names {} for {}", partitionIds, name);
        service.deletePartitions(connectorContext, name, partitionIds);
        // delete metadata
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIds, name);
        if (!partitions.isEmpty()) {
            deleteMetadatas(metacatRequestContext.getUserName(), partitions);
        }
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionIds));
    }
}