Examples with MetacatRequestContext com.netflix.metacat.common.MetacatRequestContext used on opensource projects

Example 41 with MetacatRequestContext

use of com.netflix.metacat.common.MetacatRequestContext in project metacat by Netflix.

the class MViewServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<NameDateDto> list(final QualifiedName name) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final QualifiedName viewDbName = QualifiedName.ofDatabase(name.getCatalogName(), VIEW_DB_NAME);
    final ConnectorTableService service = connectorManager.getTableService(viewDbName);
    List<QualifiedName> tableNames = Lists.newArrayList();
    try {
        final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
        tableNames = service.listNames(connectorRequestContext, viewDbName, null, null, null);
    } catch (Exception ignored) {
    // ignore. Return an empty list if database 'franklinviews' does not exist
    }
    if (!name.isDatabaseDefinition() && name.isCatalogDefinition()) {
        return tableNames.stream().map(viewName -> {
            final NameDateDto dto = new NameDateDto();
            dto.setName(viewName);
            return dto;
        }).collect(Collectors.toList());
    } else {
        final String prefix = String.format("%s_%s_", name.getDatabaseName(), MoreObjects.firstNonNull(name.getTableName(), ""));
        return tableNames.stream().filter(qualifiedTableName -> qualifiedTableName.getTableName().startsWith(prefix)).map(qualifiedTableName -> {
            final NameDateDto dto = new NameDateDto();
            dto.setName(QualifiedName.ofView(qualifiedTableName.getCatalogName(), name.getDatabaseName(), name.getTableName(), qualifiedTableName.getTableName().substring(prefix.length())));
            return dto;
        }).collect(Collectors.toList());
    }
}

Example 42 with MetacatRequestContext

use of com.netflix.metacat.common.MetacatRequestContext in project metacat by Netflix.

the class MViewServiceImpl method createAndSnapshotPartitions.

/**
 * Creates the materialized view using the schema of the give table
 * Assumes that the "franklinviews" database name already exists in the given catalog.
 */
@Override
public TableDto createAndSnapshotPartitions(final QualifiedName name, final boolean snapshot, @Nullable final String filter) {
    final TableDto result;
    // Get the table
    log.info("Get the table {}", name);
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    eventBus.postSync(new MetacatCreateMViewPreEvent(name, metacatRequestContext, this, snapshot, filter));
    final Optional<TableDto> oTable = tableService.get(name, GetTableServiceParameters.builder().includeDataMetadata(false).includeDefinitionMetadata(false).disableOnReadMetadataIntercetor(// turn off for optimization
    true).includeInfo(true).build());
    if (oTable.isPresent()) {
        final TableDto table = oTable.get();
        final String viewName = createViewName(name);
        final QualifiedName targetName = QualifiedName.ofTable(name.getCatalogName(), VIEW_DB_NAME, viewName);
        // Get the view table if it exists
        log.info("Check if the view table {} exists.", targetName);
        Optional<TableDto> oViewTable = Optional.empty();
        try {
            // read the original view back
            oViewTable = tableService.get(targetName, GetTableServiceParameters.builder().includeDataMetadata(false).includeDefinitionMetadata(false).disableOnReadMetadataIntercetor(false).includeInfo(true).build());
        } catch (NotFoundException ignored) {
        }
        if (!oViewTable.isPresent()) {
            log.info("Creating view {}.", targetName);
            result = tableService.copy(table, targetName);
        } else {
            result = oViewTable.get();
        }
        if (snapshot) {
            snapshotPartitions(name, filter);
        }
        eventBus.postAsync(new MetacatCreateMViewPostEvent(name, metacatRequestContext, this, result, snapshot, filter));
    } else {
        throw new TableNotFoundException(name);
    }
    return result;
}

Example 43 with MetacatRequestContext

use of com.netflix.metacat.common.MetacatRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method delete.

/**
 * {@inheritDoc}
 */
@Override
public void delete(final QualifiedName name, final List<String> partitionIds) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    registry.distributionSummary(this.partitionDeleteDistSummary.withTags(name.parts())).record(partitionIds.size());
    if (!tableService.exists(name)) {
        throw new TableNotFoundException(name);
    }
    if (!partitionIds.isEmpty()) {
        final PartitionsSaveRequestDto dto = new PartitionsSaveRequestDto();
        dto.setPartitionIdsForDeletes(partitionIds);
        eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, dto));
        final ConnectorPartitionService service = connectorManager.getPartitionService(name);
        // Get the partitions before calling delete
        final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto(null, partitionIds, false, true);
        final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
        final List<PartitionInfo> partitionInfos = service.getPartitions(connectorRequestContext, name, converterUtil.toPartitionListRequest(requestDto, null, null));
        List<HasMetadata> partitions = Lists.newArrayList();
        List<PartitionDto> partitionDtos = Lists.newArrayList();
        if (partitionInfos != null) {
            partitionDtos = partitionInfos.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
            partitions = new ArrayList<>(partitions);
        }
        log.info("Deleting partitions with names {} for {}", partitionIds, name);
        service.deletePartitions(connectorRequestContext, name, partitionIds);
        // delete metadata
        log.info("Deleting user metadata for partitions with names {} for {}", partitionIds, name);
        if (!partitions.isEmpty()) {
            deleteMetadatas(metacatRequestContext.getUserName(), partitions);
        }
        eventBus.postAsync(new MetacatDeleteTablePartitionPostEvent(name, metacatRequestContext, this, partitionDtos));
    }
}

Example 44 with MetacatRequestContext

use of com.netflix.metacat.common.MetacatRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method list.

/**
 * {@inheritDoc}
 */
@Override
public List<PartitionDto> list(final QualifiedName name, @Nullable final Sort sort, @Nullable final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, @Nullable final GetPartitionsRequestDto getPartitionsRequestDto) {
    // the conversion will handle getPartitionsRequestDto as null case
    final PartitionListRequest partitionListRequest = converterUtil.toPartitionListRequest(getPartitionsRequestDto, pageable, sort);
    final String filterExpression = partitionListRequest.getFilter();
    final List<String> partitionNames = partitionListRequest.getPartitionNames();
    if (Strings.isNullOrEmpty(filterExpression) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getNamesToThrowErrorOnListPartitionsWithNoFilter().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name);
    final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorRequestContext, name, partitionListRequest);
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.distributionSummary(this.partitionGetDistSummary.withTags(name.parts())).record(result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filterExpression, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}

Example 45 with MetacatRequestContext

use of com.netflix.metacat.common.MetacatRequestContext in project metacat by Netflix.

the class PartitionServiceImpl method getQualifiedNames.

/**
 * {@inheritDoc}
 */
@Override
public Map<String, List<QualifiedName>> getQualifiedNames(final List<String> uris, final boolean prefixSearch) {
    final Map<String, List<QualifiedName>> result = Maps.newConcurrentMap();
    final List<ListenableFuture<Void>> futures = Lists.newArrayList();
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    connectorManager.getPartitionServices().forEach(service -> {
        futures.add(threadServiceManager.getExecutor().submit(() -> {
            final ConnectorRequestContext connectorRequestContext = converterUtil.toConnectorContext(metacatRequestContext);
            try {
                final Map<String, List<QualifiedName>> partitionNames = service.getPartitionNames(connectorRequestContext, uris, prefixSearch);
                partitionNames.forEach((uri, subPartitionNames) -> {
                    final List<QualifiedName> existingPartitionNames = result.get(uri);
                    if (existingPartitionNames == null) {
                        result.put(uri, subPartitionNames);
                    } else {
                        existingPartitionNames.addAll(subPartitionNames);
                    }
                });
            } catch (final UnsupportedOperationException uoe) {
                log.debug("Partition service doesn't support getPartitionNames. Ignoring.");
            }
            return null;
        }));
    });
    try {
        Futures.allAsList(futures).get(1, TimeUnit.HOURS);
    } catch (Exception e) {
        Throwables.propagate(e);
    }
    return result;
}