Examples with MetacatSaveTablePartitionPostEvent com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent used on opensource projects

Example 1 with MetacatSaveTablePartitionPostEvent

use of com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent in project metacat by Netflix.

the class ElasticSearchEventHandlers method metacatSaveTablePartitionPostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatSaveTablePartitionPostEventHandler(final MetacatSaveTablePartitionPostEvent event) {
    log.debug("Received SaveTablePartitionEvent {}", event);
    this.partitionSaveEventsDelayTimer.record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS);
    if (config.isElasticSearchPublishPartitionEnabled()) {
        this.partitionSaveTimer.record(() -> {
            final List<PartitionDto> partitionDtos = event.getPartitions();
            final MetacatRequestContext context = event.getRequestContext();
            final List<ElasticSearchDoc> docs = partitionDtos.stream().map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, context.getUserName(), false)).collect(Collectors.toList());
            es.save(ElasticSearchDoc.Type.partition.name(), docs);
        });
    }
}

Example 2 with MetacatSaveTablePartitionPostEvent

use of com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent in project metacat by Netflix.

the class MetacatElasticSearchEventHandlers method metacatSaveTablePartitionPostEventHandler.

/**
     * Subscriber.
     *
     * @param event event
     */
@EventListener
public void metacatSaveTablePartitionPostEventHandler(final MetacatSaveTablePartitionPostEvent event) {
    log.debug("Received SaveTablePartitionEvent {}", event);
    registry.counter(Metrics.CounterElasticSearchPartitionSave.name()).increment();
    final List<PartitionDto> partitionDtos = event.getPartitions();
    final MetacatRequestContext context = event.getRequestContext();
    final List<ElasticSearchDoc> docs = partitionDtos.stream().map(dto -> new ElasticSearchDoc(dto.getName().toString(), dto, context.getUserName(), false)).collect(Collectors.toList());
    es.save(ElasticSearchDoc.Type.partition.name(), docs);
}

Example 3 with MetacatSaveTablePartitionPostEvent

use of com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent in project metacat by Netflix.

the class PartitionServiceImpl method updatePartitions.

/**
 * Add, delete, update partitions.
 *
 * @param service               partition service
 * @param metacatRequestContext metacat request context
 * @param dto                   partition save request dto
 * @param name                  qualified name
 * @param partitionDtos         partitions dto
 * @return partition save response dto
 */
private PartitionsSaveResponseDto updatePartitions(final ConnectorPartitionService service, final MetacatRequestContext metacatRequestContext, final PartitionsSaveRequestDto dto, final QualifiedName name, final List<PartitionDto> partitionDtos) {
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    List<PartitionDto> deletePartitionDtos = Lists.newArrayList();
    validate(name, dto);
    registry.distributionSummary(this.partitionAddDistSummary.withTags(name.parts())).record(partitionDtos.size());
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.post(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIdsForDeletes, false, true);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null), this.getTableInfo(name));
        if (deletePartitionInfos != null) {
            deletePartitionDtos = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            deletePartitions = new ArrayList<>(deletePartitions);
        }
    }
    // Save all the new and updated partitions
    eventBus.post(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions for {} ({})", name, partitionDtos.size());
    final PartitionsSaveResponseDto result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorRequestContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    final long start = registry.clock().wallTime();
    userMetadataService.saveMetadata(metacatRequestContext.getUserName(), partitionDtos, true);
    final long duration = registry.clock().wallTime() - start;
    log.info("Time taken to save user metadata for table {} is {} ms", name, duration);
    registry.timer(registry.createId(Metrics.TimerSavePartitionMetadata.getMetricName()).withTags(name.parts())).record(duration, TimeUnit.MILLISECONDS);
    // TODO: create MetacatUpdateTablePartitionEvents, only publish one partitionUpdateEvent here.
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.post(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, deletePartitionDtos));
    }
    eventBus.post(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    return result;
}

Example 4 with MetacatSaveTablePartitionPostEvent

use of com.netflix.metacat.common.server.events.MetacatSaveTablePartitionPostEvent in project metacat by Netflix.

the class MetacatServiceHelper method postPostUpdateEvent.

/**
 * Calls the right method of the event bus for the given qualified name.
 *
 * @param name                  name
 * @param metacatRequestContext context
 * @param oldDTo                dto
 * @param currentDto            dto
 */
public void postPostUpdateEvent(final QualifiedName name, final MetacatRequestContext metacatRequestContext, final BaseDto oldDTo, final BaseDto currentDto) {
    if (name.isPartitionDefinition()) {
        final List<PartitionDto> dtos = Lists.newArrayList();
        if (currentDto != null) {
            dtos.add((PartitionDto) currentDto);
        }
        // This request neither added nor updated partitions
        final PartitionsSaveResponseDto partitionsSaveResponseDto = new PartitionsSaveResponseDto();
        this.eventBus.post(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, dtos, partitionsSaveResponseDto));
    } else if (name.isViewDefinition()) {
        final MetacatUpdateMViewPostEvent event = new MetacatUpdateMViewPostEvent(name, metacatRequestContext, this, (TableDto) currentDto);
        this.eventBus.post(event);
    } else if (name.isTableDefinition()) {
        final MetacatUpdateTablePostEvent event = new MetacatUpdateTablePostEvent(name, metacatRequestContext, this, (TableDto) oldDTo, (TableDto) currentDto);
        this.eventBus.post(event);
    } else if (name.isDatabaseDefinition()) {
        this.eventBus.post(new MetacatUpdateDatabasePostEvent(name, metacatRequestContext, this));
    } else {
        throw new IllegalArgumentException(String.format("Invalid name %s", name));
    }
}