Examples with PartitionDto com.netflix.metacat.common.dto.PartitionDto used on opensource projects

Example 6 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class MetacatServiceHelper method postPreUpdateEvent.

/**
     * Calls the right method of the event bus for the given qualified name.
     *
     * @param name                  name
     * @param metacatRequestContext context
     * @param dto                   dto
     */
public void postPreUpdateEvent(final QualifiedName name, final MetacatRequestContext metacatRequestContext, final BaseDto dto) {
    if (name.isPartitionDefinition()) {
        final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto();
        if (dto != null) {
            partitionsSaveRequestDto.setPartitions(ImmutableList.of((PartitionDto) dto));
        }
        this.eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, partitionsSaveRequestDto));
    } else if (name.isTableDefinition()) {
        this.eventBus.postSync(new MetacatUpdateTablePreEvent(name, metacatRequestContext, this, (TableDto) dto, (TableDto) dto));
    } else if (name.isDatabaseDefinition()) {
        eventBus.postSync(new MetacatUpdateDatabasePreEvent(name, metacatRequestContext, this));
    } else {
        throw new IllegalArgumentException(String.format("Invalid name %s", name));
    }
}

Example 7 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class MViewServiceImpl method snapshotPartitions.

@Override
public void snapshotPartitions(@Nonnull final QualifiedName name, final String filter) {
    final List<PartitionDto> partitionDtos = partitionService.list(name, filter, null, null, null, false, false, true);
    if (partitionDtos != null && !partitionDtos.isEmpty()) {
        log.info("Snapshot partitions({}) for view {}.", partitionDtos.size(), name);
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitions(partitionDtos);
        savePartitions(name, dto, false);
    }
}

Example 8 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class PartitionServiceImpl method save.

@Override
public PartitionsSaveResponseDto save(@Nonnull final QualifiedName name, final PartitionsSaveRequestDto dto) {
    PartitionsSaveResponseDto result = new PartitionsSaveResponseDto();
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final List<PartitionDto> partitionDtos = dto.getPartitions();
    // If no partitions are passed, then return
    if (partitionDtos == null || partitionDtos.isEmpty()) {
        return result;
    }
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    registry.gauge(this.partitionAddedCountId.withTags(new HashMap<>(name.parts())), partitionDtos.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.gauge(this.partitionDeletedCountId.withTags(new HashMap<>(name.parts())), partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
        requestDto.setIncludePartitionDetails(false);
        requestDto.setPartitionNames(partitionIdsForDeletes);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        if (deletePartitionInfos != null) {
            deletePartitions = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        }
    }
    //
    // Save all the new and updated partitions
    //
    eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions({}) for {}", partitionDtos.size(), name);
    result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    userMetadataService.saveMetadatas(metacatRequestContext.getUserName(), partitionDtos, true);
    eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionIdsForDeletes));
    }
    return result;
}

Example 9 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class ElasticSearchMetacatRefresh method _processPartitions.

@SuppressWarnings("checkstyle:methodname")
private ListenableFuture<Void> _processPartitions(final List<QualifiedName> qNames) {
    final List<QualifiedName> excludeQualifiedNames = config.getElasticSearchRefreshExcludeQualifiedNames();
    final List<String> tables = elasticSearchUtil.getTableIdsByCatalogs(ElasticSearchDoc.Type.table.name(), qNames, excludeQualifiedNames);
    final List<ListenableFuture<ListenableFuture<Void>>> futures = tables.stream().map(s -> service.submit(() -> {
        final QualifiedName tableName = QualifiedName.fromString(s, false);
        final List<ListenableFuture<Void>> indexFutures = Lists.newArrayList();
        int offset = 0;
        int count;
        final Sort sort;
        if ("s3".equals(tableName.getCatalogName()) || "aegisthus".equals(tableName.getCatalogName())) {
            sort = new Sort("id", SortOrder.ASC);
        } else {
            sort = new Sort("part_id", SortOrder.ASC);
        }
        final Pageable pageable = new Pageable(10000, offset);
        do {
            final List<PartitionDto> partitionDtos = partitionService.list(tableName, null, null, sort, pageable, true, true, true);
            count = partitionDtos.size();
            if (!partitionDtos.isEmpty()) {
                final List<List<PartitionDto>> partitionedPartitionDtos = Lists.partition(partitionDtos, 1000);
                partitionedPartitionDtos.forEach(subPartitionsDtos -> indexFutures.add(indexPartitionDtos(tableName, subPartitionsDtos)));
                offset = offset + count;
                pageable.setOffset(offset);
            }
        } while (count == 10000);
        return Futures.transform(Futures.successfulAsList(indexFutures), Functions.constant((Void) null));
    })).collect(Collectors.toList());
    final ListenableFuture<Void> processPartitionsFuture = Futures.transformAsync(Futures.successfulAsList(futures), input -> {
        final List<ListenableFuture<Void>> inputFuturesWithoutNulls = input.stream().filter(NOT_NULL).collect(Collectors.toList());
        return Futures.transform(Futures.successfulAsList(inputFuturesWithoutNulls), Functions.constant(null));
    });
    return Futures.transformAsync(processPartitionsFuture, input -> {
        elasticSearchUtil.refresh();
        final List<ListenableFuture<Void>> cleanUpFutures = tables.stream().map(s -> service.submit(() -> partitionsCleanUp(QualifiedName.fromString(s, false), excludeQualifiedNames))).collect(Collectors.toList());
        return Futures.transform(Futures.successfulAsList(cleanUpFutures), Functions.constant(null));
    });
}

Example 10 with PartitionDto

use of com.netflix.metacat.common.dto.PartitionDto in project metacat by Netflix.

the class SNSNotificationServiceImpl method notifyOfPartitionAddition.

/**
     * {@inheritDoc}
     */
@Override
@EventListener
public void notifyOfPartitionAddition(@NonNull final MetacatSaveTablePartitionPostEvent event) {
    log.debug("Received SaveTablePartitionPostEvent {}", event);
    final String name = event.getName().toString();
    final long timestamp = event.getRequestContext().getTimestamp();
    final String requestId = event.getRequestContext().getId();
    AddPartitionMessage message = null;
    try {
        for (final PartitionDto partition : event.getPartitions()) {
            message = new AddPartitionMessage(UUID.randomUUID().toString(), timestamp, requestId, name, partition);
            this.publishNotification(this.partitionTopicArn, message);
            log.debug("Published create partition message {} on {}", message, this.partitionTopicArn);
            this.registry.counter(this.registry.createId(Metrics.CounterSNSNotificationPartitionAdd.name()).withTags(Metrics.statusSuccessMap)).increment();
        }
    } catch (final Exception e) {
        this.handleException(event.getName(), "Unable to publish partition creation notification", Metrics.CounterSNSNotificationPartitionAdd.name(), message, e);
    }
    UpdateTablePartitionsMessage tableMessage = null;
    try {
        // Publish a global message stating how many partitions were updated for the table to the table topic
        tableMessage = new UpdateTablePartitionsMessage(UUID.randomUUID().toString(), timestamp, requestId, name, new TablePartitionsUpdatePayload(event.getPartitions().size(), 0));
        this.publishNotification(this.tableTopicArn, tableMessage);
        this.registry.counter(this.registry.createId(Metrics.CounterSNSNotificationTablePartitionAdd.name()).withTags(Metrics.statusSuccessMap)).increment();
    } catch (final Exception e) {
        this.handleException(event.getName(), "Unable to publish table partition add notification", Metrics.CounterSNSNotificationTablePartitionAdd.name(), tableMessage, e);
    }
}