Examples with PartitionInfo com.netflix.metacat.common.server.connectors.model.PartitionInfo used on opensource projects

Example 11 with PartitionInfo

use of com.netflix.metacat.common.server.connectors.model.PartitionInfo in project metacat by Netflix.

the class HiveConnectorPartitionService method savePartitions.

/**
     * {@inheritDoc}.
     */
@Override
public PartitionsSaveResponse savePartitions(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName, @Nonnull @NonNull final PartitionsSaveRequest partitionsSaveRequest) {
    final String databasename = tableName.getDatabaseName();
    final String tablename = tableName.getTableName();
    // New partitions
    final List<Partition> hivePartitions = Lists.newArrayList();
    try {
        final Table table = metacatHiveClient.getTableByName(databasename, tablename);
        final List<PartitionInfo> partitionInfos = partitionsSaveRequest.getPartitions();
        // New partition ids
        final List<String> addedPartitionIds = Lists.newArrayList();
        // Updated partition ids
        final List<String> existingPartitionIds = Lists.newArrayList();
        // Existing partitions
        final List<Partition> existingHivePartitions = Lists.newArrayList();
        // Existing partition map
        Map<String, Partition> existingPartitionMap = Collections.emptyMap();
        if (partitionsSaveRequest.getCheckIfExists()) {
            final List<String> partitionNames = partitionInfos.stream().map(partition -> {
                final String partitionName = partition.getName().getPartitionName();
                PartitionUtil.validatePartitionName(partitionName, getPartitionKeys(table.getPartitionKeys()));
                return partitionName;
            }).collect(Collectors.toList());
            existingPartitionMap = getPartitionsByNames(table, partitionNames);
        }
        final TableInfo tableInfo = hiveMetacatConverters.toTableInfo(tableName, table);
        for (PartitionInfo partitionInfo : partitionInfos) {
            final String partitionName = partitionInfo.getName().getPartitionName();
            final Partition hivePartition = existingPartitionMap.get(partitionName);
            if (hivePartition == null) {
                addedPartitionIds.add(partitionName);
                hivePartitions.add(hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo));
            } else {
                //unless we alterifExists
                if (partitionsSaveRequest.getAlterIfExists()) {
                    final Partition existingPartition = hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo);
                    existingPartitionIds.add(partitionName);
                    existingPartition.setParameters(hivePartition.getParameters());
                    existingPartition.setCreateTime(hivePartition.getCreateTime());
                    existingPartition.setLastAccessTime(hivePartition.getLastAccessTime());
                    existingHivePartitions.add(existingPartition);
                }
            }
        }
        final Set<String> deletePartitionIds = Sets.newHashSet();
        if (!partitionsSaveRequest.getAlterIfExists()) {
            deletePartitionIds.addAll(existingPartitionIds);
        }
        if (partitionsSaveRequest.getPartitionIdsForDeletes() != null) {
            deletePartitionIds.addAll(partitionsSaveRequest.getPartitionIdsForDeletes());
        }
        if (partitionsSaveRequest.getAlterIfExists() && !existingHivePartitions.isEmpty()) {
            copyTableSdToPartitionSd(existingHivePartitions, table);
            metacatHiveClient.alterPartitions(databasename, tablename, existingHivePartitions);
        }
        copyTableSdToPartitionSd(hivePartitions, table);
        metacatHiveClient.addDropPartitions(databasename, tablename, hivePartitions, Lists.newArrayList(deletePartitionIds));
        final PartitionsSaveResponse result = new PartitionsSaveResponse();
        result.setAdded(addedPartitionIds);
        result.setUpdated(existingPartitionIds);
        return result;
    } catch (NoSuchObjectException exception) {
        if (exception.getMessage() != null && exception.getMessage().startsWith("Partition doesn't exist")) {
            throw new PartitionNotFoundException(tableName, "", exception);
        } else {
            throw new TableNotFoundException(tableName, exception);
        }
    } catch (MetaException | InvalidObjectException exception) {
        throw new InvalidMetaException("One or more partitions are invalid.", exception);
    } catch (AlreadyExistsException e) {
        final List<String> ids = getFakePartitionName(hivePartitions);
        throw new PartitionAlreadyExistsException(tableName, ids, e);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed savePartitions hive table %s", tableName), exception);
    }
}

Example 12 with PartitionInfo

use of com.netflix.metacat.common.server.connectors.model.PartitionInfo in project metacat by Netflix.

the class PartitionServiceImpl method delete.

/**
 * {@inheritDoc}
 */
@Override
public void delete(final QualifiedName name, final List<String> partitionIds) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIds.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    if (!partitionIds.isEmpty()) {
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitionIdsForDeletes(partitionIds);
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        final ConnectorPartitionService service = connectorManager.getPartitionService(name);
        // Get the partitions before calling delete
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIds, false, true);
        final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
        final List<PartitionInfo> partitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        List<HasMetadata> partitions = Lists.newArrayList();
        List<PartitionDto> partitionDtos = Lists.newArrayList();
        if (partitionInfos != null) {
            partitionDtos = partitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            partitions = new ArrayList<>(partitions);
        }
        log.info("Deleting partitions with names {} for {}", partitionIds, name);
        service.deletePartitions(connectorRequestContext, name, partitionIds);
        // delete metadata
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIds, name);
        if (!partitions.isEmpty()) {
            deleteMetadatas(metacatRequestContext.getUserName(), partitions);
        }
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos));
    }
}

Example 13 with PartitionInfo

use of com.netflix.metacat.common.server.connectors.model.PartitionInfo in project metacat by Netflix.

the class PartitionServiceImpl method updatePartitions.

/**
 * Add, delete, update partitions.
 *
 * @param service               partition service
 * @param metacatRequestContext metacat request context
 * @param dto                   partition save request dto
 * @param name                  qualified name
 * @param partitionDtos         partitions dto
 * @return partition save response dto
 */
private PartitionsSaveResponseDto updatePartitions(final ConnectorPartitionService service, final MetacatRequestContext metacatRequestContext, final PartitionsSaveRequestDto dto, final QualifiedName name, final List<PartitionDto> partitionDtos) {
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    List<PartitionDto> deletePartitionDtos = Lists.newArrayList();
    registry.distributionSummary(this.partitionAddDistSummary.withTags(name.parts())).record(partitionDtos.size());
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIdsForDeletes, false, true);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        if (deletePartitionInfos != null) {
            deletePartitionDtos = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            deletePartitions = new ArrayList<>(deletePartitions);
        }
    }
    // Save all the new and updated partitions
    eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions for {} ({})", name, partitionDtos.size());
    final PartitionsSaveResponseDto result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorRequestContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    final long start = registry.clock().wallTime();
    userMetadataService.saveMetadata(metacatRequestContext.getUserName(), partitionDtos, true);
    final long duration = registry.clock().wallTime() - start;
    log.info("Time taken to save user metadata for table {} is {} ms", name, duration);
    registry.timer(registry.createId(Metrics.TimerSavePartitionMetadata.getMetricName()).withTags(name.parts())).record(duration, TimeUnit.MILLISECONDS);
    // TODO: create MetacatUpdateTablePartitionEvents, only publish one partitionUpdateEvent here.
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, deletePartitionDtos));
    }
    eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    return result;
}

Example 14 with PartitionInfo

use of com.netflix.metacat.common.server.connectors.model.PartitionInfo in project metacat by Netflix.

the class PartitionServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionDto> list(final QualifiedName name, @Nullable final Sort sort, @Nullable final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, @Nullable final GetPartitionsRequestDto getPartitionsRequestDto) {
    // the conversion will handle getPartitionsRequestDto as null case
    final PartitionListRequest partitionListRequest = converterUtil.toPartitionListRequest(getPartitionsRequestDto, pageable, sort);
    final String filterExpression = partitionListRequest.getFilter();
    final List<String> partitionNames = partitionListRequest.getPartitionNames();
    if (Strings.isNullOrEmpty(filterExpression) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getNamesToThrowErrorOnListPartitionsWithNoFilter().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name);
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorRequestContext, name, partitionListRequest);
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.distributionSummary(this.partitionGetDistSummary.withTags(name.parts())).record(result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filterExpression, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}

Example 15 with PartitionInfo

use of com.netflix.metacat.common.server.connectors.model.PartitionInfo in project metacat by Netflix.

the class S3ConnectorPartitionService method savePartitions.

@Override
public PartitionsSaveResponse savePartitions(@Nonnull final ConnectorRequestContext context, @Nonnull final QualifiedName tableName, @Nonnull final PartitionsSaveRequest partitionsSaveRequest) {
    log.debug("Start: Save partitions for table {}", tableName);
    // Table
    final Table table = getTable(tableName);
    // New partition ids
    final List<String> addedPartitionIds = Lists.newArrayList();
    // Updated partition ids
    final List<String> existingPartitionIds = Lists.newArrayList();
    // 
    Map<String, Partition> existingPartitionMap = Maps.newHashMap();
    if (partitionsSaveRequest.getCheckIfExists()) {
        final List<String> partitionNames = partitionsSaveRequest.getPartitions().stream().map(partition -> {
            final String partitionName = partition.getName().getPartitionName();
            PartitionUtil.validatePartitionName(partitionName, infoConverter.partitionKeys(table));
            return partitionName;
        }).collect(Collectors.toList());
        existingPartitionMap = getPartitionsByNames(table.getId(), partitionNames);
    }
    // New partitions
    final List<Partition> s3Partitions = Lists.newArrayList();
    for (PartitionInfo partition : partitionsSaveRequest.getPartitions()) {
        final String partitionName = partition.getName().getPartitionName();
        final Partition s3Partition = existingPartitionMap.get(partitionName);
        if (s3Partition == null) {
            addedPartitionIds.add(partitionName);
            s3Partitions.add(infoConverter.toPartition(table, partition));
        } else {
            final String partitionUri = infoConverter.getUri(partition);
            final String s3PartitionUri = s3Partition.getUri();
            if (partitionUri != null && !partitionUri.equals(s3PartitionUri)) {
                s3Partition.setUri(partitionUri);
                existingPartitionIds.add(partitionName);
                s3Partitions.add(s3Partition);
            }
        }
    }
    final List<String> partitionIdsForDeletes = partitionsSaveRequest.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        partitionDao.deleteByNames(catalogName, tableName.getDatabaseName(), tableName.getTableName(), partitionIdsForDeletes);
    }
    partitionDao.save(s3Partitions);
    log.debug("End: Save partitions for table {}", tableName);
    return PartitionsSaveResponse.builder().added(addedPartitionIds).updated(existingPartitionIds).build();
}