Search in sources :

Example 1 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method getPartitionKeys.

@Override
public List<String> getPartitionKeys(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable) {
    List<String> result = Lists.newArrayList();
    if (tableService.exists(name)) {
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setFilter(filter);
        requestDto.setPartitionNames(partitionNames);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        try {
            result = service.getPartitionKeys(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
        } catch (final UnsupportedOperationException uoe) {
            log.debug("Catalog {} doesn't support getPartitionKeys. Ignoring.", name.getCatalogName());
        }
    }
    return result;
}
Also used : MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) GetPartitionsRequestDto(com.netflix.metacat.common.dto.GetPartitionsRequestDto)

Example 2 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method save.

@Override
public PartitionsSaveResponseDto save(@Nonnull final QualifiedName name, final PartitionsSaveRequestDto dto) {
    PartitionsSaveResponseDto result = new PartitionsSaveResponseDto();
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final List<PartitionDto> partitionDtos = dto.getPartitions();
    // If no partitions are passed, then return
    if (partitionDtos == null || partitionDtos.isEmpty()) {
        return result;
    }
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    registry.gauge(this.partitionAddedCountId.withTags(new HashMap<>(name.parts())), partitionDtos.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.gauge(this.partitionDeletedCountId.withTags(new HashMap<>(name.parts())), partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setIncludePartitionDetails(false);
        requestDto.setPartitionNames(partitionIdsForDeletes);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        if (deletePartitionInfos != null) {
            deletePartitions = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        }
    }
    //
    // Save all the new and updated partitions
    //
    eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions({}) for {}", partitionDtos.size(), name);
    result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    userMetadataService.saveMetadatas(metacatRequestContext.getUserName(), partitionDtos, true);
    eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionIdsForDeletes));
    }
    return result;
}
Also used : MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) HashMap(java.util.HashMap) MetacatSaveTablePartitionPreEvent(com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPreEvent) GetPartitionsRequestDto(com.netflix.metacat.common.dto.GetPartitionsRequestDto) MetacatDeleteTablePartitionPreEvent(com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPreEvent) MetacatDeleteTablePartitionPostEvent(com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPostEvent) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) MetacatSaveTablePartitionPostEvent(com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) PartitionDto(com.netflix.metacat.common.dto.PartitionDto) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo) PartitionsSaveResponseDto(com.netflix.metacat.common.dto.PartitionsSaveResponseDto)

Example 3 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method count.

@Override
public Integer count(final QualifiedName name) {
    Integer result = 0;
    if (tableService.exists(name)) {
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        result = service.getPartitionCount(connectorContext, name);
    }
    return result;
}
Also used : MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService)

Example 4 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method getPartitionUris.

@Override
public List<String> getPartitionUris(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable) {
    List<String> result = Lists.newArrayList();
    if (tableService.exists(name)) {
        final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setFilter(filter);
        requestDto.setPartitionNames(partitionNames);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        try {
            result = service.getPartitionUris(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
        } catch (final UnsupportedOperationException uoe) {
            log.info("Catalog {} doesn't support getPartitionUris. Ignoring.", name.getCatalogName());
        }
    }
    return result;
}
Also used : MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) GetPartitionsRequestDto(com.netflix.metacat.common.dto.GetPartitionsRequestDto)

Example 5 with ConnectorPartitionService

use of com.netflix.metacat.common.server.connectors.ConnectorPartitionService in project metacat by Netflix.

the class PartitionServiceImpl method delete.

@Override
public void delete(final QualifiedName name, final List<String> partitionIds) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    registry.gauge(this.partitionDeletedCountId.withTags(new HashMap<>(name.parts())), partitionIds.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    if (!partitionIds.isEmpty()) {
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitionIdsForDeletes(partitionIds);
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
        // Get the partitions before calling delete
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setIncludePartitionDetails(false);
        requestDto.setPartitionNames(partitionIds);
        final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
        final List<PartitionInfo> partitionInfos = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        List<HasMetadata> partitions = Lists.newArrayList();
        if (partitionInfos != null) {
            partitions = partitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        }
        log.info("Deleting partitions with names {} for {}", partitionIds, name);
        service.deletePartitions(connectorContext, name, partitionIds);
        // delete metadata
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIds, name);
        if (!partitions.isEmpty()) {
            deleteMetadatas(metacatRequestContext.getUserName(), partitions);
        }
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionIds));
    }
}
Also used : MetacatRequestContext(com.netflix.metacat.common.MetacatRequestContext) HasMetadata(com.netflix.metacat.common.dto.HasMetadata) HashMap(java.util.HashMap) GetPartitionsRequestDto(com.netflix.metacat.common.dto.GetPartitionsRequestDto) MetacatDeleteTablePartitionPreEvent(com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPreEvent) MetacatDeleteTablePartitionPostEvent(com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPostEvent) PartitionsSaveRequestDto(com.netflix.metacat.common.dto.PartitionsSaveRequestDto) TableNotFoundException(com.netflix.metacat.common.server.connectors.exception.TableNotFoundException) ConnectorContext(com.netflix.metacat.common.server.connectors.ConnectorContext) ConnectorPartitionService(com.netflix.metacat.common.server.connectors.ConnectorPartitionService) PartitionInfo(com.netflix.metacat.common.server.connectors.model.PartitionInfo)

Aggregations

MetacatRequestContext (com.netflix.metacat.common.MetacatRequestContext)7 ConnectorContext (com.netflix.metacat.common.server.connectors.ConnectorContext)7 ConnectorPartitionService (com.netflix.metacat.common.server.connectors.ConnectorPartitionService)7 GetPartitionsRequestDto (com.netflix.metacat.common.dto.GetPartitionsRequestDto)6 TableNotFoundException (com.netflix.metacat.common.server.connectors.exception.TableNotFoundException)4 PartitionInfo (com.netflix.metacat.common.server.connectors.model.PartitionInfo)4 HashMap (java.util.HashMap)4 HasMetadata (com.netflix.metacat.common.dto.HasMetadata)3 PartitionDto (com.netflix.metacat.common.dto.PartitionDto)3 MetacatDeleteTablePartitionPostEvent (com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPostEvent)3 MetacatDeleteTablePartitionPreEvent (com.netflix.metacat.common.server.events.MetacatDeleteTablePartitionPreEvent)3 ObjectNode (com.fasterxml.jackson.databind.node.ObjectNode)2 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)2 QualifiedName (com.netflix.metacat.common.QualifiedName)2 PartitionsSaveRequestDto (com.netflix.metacat.common.dto.PartitionsSaveRequestDto)2 PartitionsSaveResponseDto (com.netflix.metacat.common.dto.PartitionsSaveResponseDto)2 MetacatSaveTablePartitionPostEvent (com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent)2 MetacatSaveTablePartitionPreEvent (com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPreEvent)2 Map (java.util.Map)2 Strings (com.google.common.base.Strings)1