Examples with ConnectorRequestContext com.netflix.metacat.common.server.connectors.ConnectorRequestContext used on opensource projects

Example 16 with ConnectorRequestContext

use of com.netflix.metacat.common.server.connectors.ConnectorRequestContext in project metacat by Netflix.

the class DatabaseServiceImpl method delete.

/**
 * {@inheritDoc}
 */
@Override
public void delete(final QualifiedName name) {
    validate(name);
    log.info("Dropping schema {}", name);
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final DatabaseDto dto = get(name, GetDatabaseServiceParameters.builder().disableOnReadMetadataIntercetor(false).includeUserMetadata(true).includeTableNames(true).build());
    eventBus.postSync(new MetacatDeleteDatabasePreEvent(name, metacatRequestContext, this, dto));
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    connectorManager.getDatabaseService(name).delete(connectorRequestContext, name);
    // Delete definition metadata if it exists
    if (userMetadataService.getDefinitionMetadata(name).isPresent()) {
        log.info("Deleting user metadata for schema {}", name);
        userMetadataService.deleteDefinitionMetadata(ImmutableList.of(name));
    }
    eventBus.postAsync(new MetacatDeleteDatabasePostEvent(name, metacatRequestContext, this, dto));
}

Example 17 with ConnectorRequestContext

use of com.netflix.metacat.common.server.connectors.ConnectorRequestContext in project metacat by Netflix.

the class MViewServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<NameDateDto> list(final QualifiedName name) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final QualifiedName viewDbName = QualifiedName.ofDatabase(name.getCatalogName(), VIEW_DB_NAME);
    final ConnectorTableService service = connectorManager.getTableService(viewDbName);
    List<QualifiedName> tableNames = Lists.newArrayList();
    try {
        final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
        tableNames = service.listNames(connectorRequestContext, viewDbName, null, null, null);
    } catch (Exception ignored) {
    // ignore. Return an empty list if database 'franklinviews' does not exist
    }
    if (!name.isDatabaseDefinition() && name.isCatalogDefinition()) {
        return tableNames.stream().map(viewName -> {
            final NameDateDto dto = new NameDateDto();
            dto.setName(viewName);
            return dto;
        }).collect(Collectors.toList());
    } else {
        final String prefix = String.format("%s_%s_", name.getDatabaseName(), MoreObjects.firstNonNull(name.getTableName(), ""));
        return tableNames.stream().filter(qualifiedTableName -> qualifiedTableName.getTableName().startsWith(prefix)).map(qualifiedTableName -> {
            final NameDateDto dto = new NameDateDto();
            dto.setName(QualifiedName.ofView(qualifiedTableName.getCatalogName(), name.getDatabaseName(), name.getTableName(), qualifiedTableName.getTableName().substring(prefix.length())));
            return dto;
        }).collect(Collectors.toList());
    }
}

Example 18 with ConnectorRequestContext

use of com.netflix.metacat.common.server.connectors.ConnectorRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method delete.

/**
 * {@inheritDoc}
 */
@Override
public void delete(final QualifiedName name, final List<String> partitionIds) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIds.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    if (!partitionIds.isEmpty()) {
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitionIdsForDeletes(partitionIds);
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        final ConnectorPartitionService service = connectorManager.getPartitionService(name);
        // Get the partitions before calling delete
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIds, false, true);
        final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
        final List<PartitionInfo> partitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        List<HasMetadata> partitions = Lists.newArrayList();
        List<PartitionDto> partitionDtos = Lists.newArrayList();
        if (partitionInfos != null) {
            partitionDtos = partitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            partitions = new ArrayList<>(partitions);
        }
        log.info("Deleting partitions with names {} for {}", partitionIds, name);
        service.deletePartitions(connectorRequestContext, name, partitionIds);
        // delete metadata
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIds, name);
        if (!partitions.isEmpty()) {
            deleteMetadatas(metacatRequestContext.getUserName(), partitions);
        }
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos));
    }
}

Example 19 with ConnectorRequestContext

use of com.netflix.metacat.common.server.connectors.ConnectorRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method updatePartitions.

/**
 * Add, delete, update partitions.
 *
 * @param service               partition service
 * @param metacatRequestContext metacat request context
 * @param dto                   partition save request dto
 * @param name                  qualified name
 * @param partitionDtos         partitions dto
 * @return partition save response dto
 */
private PartitionsSaveResponseDto updatePartitions(final ConnectorPartitionService service, final MetacatRequestContext metacatRequestContext, final PartitionsSaveRequestDto dto, final QualifiedName name, final List<PartitionDto> partitionDtos) {
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    List<HasMetadata> deletePartitions = Lists.newArrayList();
    List<PartitionDto> deletePartitionDtos = Lists.newArrayList();
    registry.distributionSummary(this.partitionAddDistSummary.withTags(name.parts())).record(partitionDtos.size());
    final List<String> partitionIdsForDeletes = dto.getPartitionIdsForDeletes();
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIdsForDeletes.size());
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIdsForDeletes, false, true);
        final List<PartitionInfo> deletePartitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        if (deletePartitionInfos != null) {
            deletePartitionDtos = deletePartitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            deletePartitions = new ArrayList<>(deletePartitions);
        }
    }
    // Save all the new and updated partitions
    eventBus.postSync(new MetacatSaveTablePartitionPreEvent(name, metacatRequestContext, this, dto));
    log.info("Saving partitions for {} ({})", name, partitionDtos.size());
    final PartitionsSaveResponseDto result = converterUtil.toPartitionsSaveResponseDto(service.savePartitions(connectorRequestContext, name, converterUtil.toPartitionsSaveRequest(dto)));
    // Save metadata
    log.info("Saving user metadata for partitions for {}", name);
    // delete metadata
    if (!deletePartitions.isEmpty()) {
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIdsForDeletes, name);
        deleteMetadatas(metacatRequestContext.getUserName(), deletePartitions);
    }
    final long start = registry.clock().wallTime();
    userMetadataService.saveMetadata(metacatRequestContext.getUserName(), partitionDtos, true);
    final long duration = registry.clock().wallTime() - start;
    log.info("Time taken to save user metadata for table {} is {} ms", name, duration);
    registry.timer(registry.createId(Metrics.TimerSavePartitionMetadata.getMetricName()).withTags(name.parts())).record(duration, TimeUnit.MILLISECONDS);
    // TODO: create MetacatUpdateTablePartitionEvents, only publish one partitionUpdateEvent here.
    if (partitionIdsForDeletes != null && !partitionIdsForDeletes.isEmpty()) {
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, deletePartitionDtos));
    }
    eventBus.postAsync(new MetacatSaveTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos, result));
    return result;
}

Example 20 with ConnectorRequestContext

use of com.netflix.metacat.common.server.connectors.ConnectorRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionDto> list(final QualifiedName name, @Nullable final Sort sort, @Nullable final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, @Nullable final GetPartitionsRequestDto getPartitionsRequestDto) {
    // the conversion will handle getPartitionsRequestDto as null case
    final PartitionListRequest partitionListRequest = converterUtil.toPartitionListRequest(getPartitionsRequestDto, pageable, sort);
    final String filterExpression = partitionListRequest.getFilter();
    final List<String> partitionNames = partitionListRequest.getPartitionNames();
    if (Strings.isNullOrEmpty(filterExpression) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getNamesToThrowErrorOnListPartitionsWithNoFilter().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name);
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorRequestContext, name, partitionListRequest);
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.distributionSummary(this.partitionGetDistSummary.withTags(name.parts())).record(result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filterExpression, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}