Examples with TableDto com.netflix.metacat.common.dto.TableDto used on opensource projects

Example 31 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class ElasticSearchMetacatRefresh method deleteUnmarkedEntities.

private void deleteUnmarkedEntities(final List<QualifiedName> qNames, final List<QualifiedName> excludeQualifiedNames) {
    log.info("Start: Delete unmarked entities");
    //
    // get unmarked qualified names
    // check if it not exists
    // delete
    //
    elasticSearchUtil.refresh();
    final MetacatRequestContext context = new MetacatRequestContext("admin", "metacat-refresh", null, null, null);
    final List<DatabaseDto> unmarkedDatabaseDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("database", qNames, refreshMarker, excludeQualifiedNames, DatabaseDto.class);
    if (!unmarkedDatabaseDtos.isEmpty()) {
        if (unmarkedDatabaseDtos.size() <= config.getElasticSearchThresholdUnmarkedDatabasesDelete()) {
            log.info("Start: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
            final List<String> unmarkedDatabaseNames = Lists.newArrayList();
            final List<DatabaseDto> deleteDatabaseDtos = unmarkedDatabaseDtos.stream().filter(databaseDto -> {
                boolean result = false;
                try {
                    unmarkedDatabaseNames.add(databaseDto.getName().toString());
                    final DatabaseDto dto = databaseService.get(databaseDto.getName(), false);
                    if (dto == null) {
                        result = true;
                    }
                } catch (DatabaseNotFoundException ignored) {
                    result = true;
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked databases({}): {}", unmarkedDatabaseNames.size(), unmarkedDatabaseNames);
            log.info("Deleting databases({})", deleteDatabaseDtos.size());
            if (!deleteDatabaseDtos.isEmpty()) {
                final List<QualifiedName> deleteDatabaseQualifiedNames = deleteDatabaseDtos.stream().map(DatabaseDto::getName).collect(Collectors.toList());
                final List<String> deleteDatabaseNames = deleteDatabaseQualifiedNames.stream().map(QualifiedName::toString).collect(Collectors.toList());
                log.info("Deleting databases({}): {}", deleteDatabaseNames.size(), deleteDatabaseNames);
                userMetadataService.deleteDefinitionMetadatas(deleteDatabaseQualifiedNames);
                elasticSearchUtil.softDelete("database", deleteDatabaseNames, context);
            }
            log.info("End: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
        } else {
            log.info("Count of unmarked databases({}) is more than the threshold {}", unmarkedDatabaseDtos.size(), config.getElasticSearchThresholdUnmarkedDatabasesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedDatabaseThreshholdReached.name())).increment();
        }
    }
    final List<TableDto> unmarkedTableDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("table", qNames, refreshMarker, excludeQualifiedNames, TableDto.class);
    if (!unmarkedTableDtos.isEmpty()) {
        if (unmarkedTableDtos.size() <= config.getElasticSearchThresholdUnmarkedTablesDelete()) {
            log.info("Start: Delete unmarked tables({})", unmarkedTableDtos.size());
            final List<String> unmarkedTableNames = Lists.newArrayList();
            final List<TableDto> deleteTableDtos = unmarkedTableDtos.stream().filter(tableDto -> {
                boolean result = false;
                try {
                    unmarkedTableNames.add(tableDto.getName().toString());
                    final Optional<TableDto> dto = tableService.get(tableDto.getName(), false);
                    if (!dto.isPresent()) {
                        result = true;
                    }
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked tables({}): {}", unmarkedTableNames.size(), unmarkedTableNames);
            log.info("Deleting tables({}): {}", deleteTableDtos.size());
            if (!deleteTableDtos.isEmpty()) {
                final List<String> deleteTableNames = deleteTableDtos.stream().map(dto -> dto.getName().toString()).collect(Collectors.toList());
                log.info("Deleting tables({}): {}", deleteTableNames.size(), deleteTableNames);
                userMetadataService.deleteMetadatas("admin", Lists.newArrayList(deleteTableDtos));
                // Publish event. Elasticsearch event handler will take care of updating the index already
                // TODO: Re-evaluate events vs. direct calls for these types of situations like in Genie
                deleteTableDtos.forEach(tableDto -> {
                    tagService.delete(tableDto.getName(), false);
                    this.eventBus.postAsync(new MetacatDeleteTablePostEvent(tableDto.getName(), context, this, tableDto));
                });
            }
            log.info("End: Delete unmarked tables({})", unmarkedTableDtos.size());
        } else {
            log.info("Count of unmarked tables({}) is more than the threshold {}", unmarkedTableDtos.size(), config.getElasticSearchThresholdUnmarkedTablesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedTableThreshholdReached.name())).increment();
        }
    }
    log.info("End: Delete unmarked entities");
}

Example 32 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class TableServiceImpl method get.

@Override
public Optional<TableDto> get(@Nonnull final QualifiedName name, final boolean includeInfo, final boolean includeDefinitionMetadata, final boolean includeDataMetadata) {
    validate(name);
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final ConnectorTableService service = connectorManager.getTableService(name.getCatalogName());
    final TableDto table;
    if (includeInfo) {
        try {
            table = converterUtil.toTableDto(service.get(connectorContext, name));
        } catch (NotFoundException ignored) {
            return Optional.empty();
        }
    } else {
        table = new TableDto();
        table.setName(name);
    }
    if (includeDefinitionMetadata) {
        final Optional<ObjectNode> definitionMetadata = userMetadataService.getDefinitionMetadata(name);
        if (definitionMetadata.isPresent()) {
            table.setDefinitionMetadata(definitionMetadata.get());
        }
    }
    if (includeDataMetadata) {
        TableDto dto = table;
        if (!includeInfo) {
            try {
                dto = converterUtil.toTableDto(service.get(connectorContext, name));
            } catch (NotFoundException ignored) {
            }
        }
        if (dto != null && dto.getSerde() != null) {
            final Optional<ObjectNode> dataMetadata = userMetadataService.getDataMetadata(dto.getSerde().getUri());
            if (dataMetadata.isPresent()) {
                table.setDataMetadata(dataMetadata.get());
            }
        }
    }
    return Optional.of(table);
}

Example 33 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class MViewServiceImpl method getOpt.

@Override
public Optional<TableDto> getOpt(@Nonnull final QualifiedName name) {
    final QualifiedName viewQName = QualifiedName.ofTable(name.getCatalogName(), VIEW_DB_NAME, createViewName(name));
    final Optional<TableDto> result = tableService.get(viewQName, false);
    //
    if (result.isPresent()) {
        final TableDto table = result.get();
        table.setName(name);
        final QualifiedName tableName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), name.getTableName());
        final Optional<ObjectNode> definitionMetadata = userMetadataService.getDefinitionMetadata(tableName);
        if (definitionMetadata.isPresent()) {
            userMetadataService.populateMetadata(table, definitionMetadata.get(), null);
        }
    }
    return result;
}

Example 34 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class PartitionController method savePartitions.

/**
 * Add/update partitions to the given table.
 *
 * @param catalogName              catalog name
 * @param databaseName             database name
 * @param tableName                table name
 * @param partitionsSaveRequestDto partition request containing the list of partitions to be added/updated
 * @return Response with the number of partitions added/updated
 */
@RequestMapping(method = RequestMethod.POST, path = "/catalog/{catalog-name}/database/{database-name}/table/{table-name}", consumes = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.CREATED)
@ApiOperation(position = 5, value = "Add/update partitions to the given table", notes = "Add/update partitions to the given table")
@ApiResponses({ @ApiResponse(code = HttpURLConnection.HTTP_CREATED, message = "The partitions were successfully saved"), @ApiResponse(code = HttpURLConnection.HTTP_NOT_FOUND, message = "The requested catalog or database or table cannot be located") })
@Override
public PartitionsSaveResponseDto savePartitions(@ApiParam(value = "The name of the catalog", required = true) @PathVariable("catalog-name") final String catalogName, @ApiParam(value = "The name of the database", required = true) @PathVariable("database-name") final String databaseName, @ApiParam(value = "The name of the table", required = true) @PathVariable("table-name") final String tableName, @ApiParam(value = "Request containing the list of partitions", required = true) @RequestBody final PartitionsSaveRequestDto partitionsSaveRequestDto) {
    final QualifiedName name = QualifiedName.ofTable(catalogName, databaseName, tableName);
    return this.requestWrapper.processRequest(name, "saveTablePartition", () -> {
        final PartitionsSaveResponseDto result;
        if (partitionsSaveRequestDto.getPartitions() == null || partitionsSaveRequestDto.getPartitions().isEmpty()) {
            result = new PartitionsSaveResponseDto();
        } else {
            result = this.partitionService.save(name, partitionsSaveRequestDto);
            // This metadata is actually for the table, if it is present update that
            if (partitionsSaveRequestDto.getDefinitionMetadata() != null || partitionsSaveRequestDto.getDataMetadata() != null) {
                final TableDto dto = new TableDto();
                dto.setName(name);
                dto.setDefinitionMetadata(partitionsSaveRequestDto.getDefinitionMetadata());
                dto.setDataMetadata(partitionsSaveRequestDto.getDataMetadata());
                this.v1.updateTable(catalogName, databaseName, tableName, dto);
            }
        }
        return result;
    });
}

Example 35 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class TagController method removeTableTags.

/**
 * Remove the tags from the given table.
 *
 * @param catalogName  catalog name
 * @param databaseName database name
 * @param tableName    table name
 * @param deleteAll    True if all tags need to be removed
 * @param tags         Tags to be removed from the given table
 */
@RequestMapping(method = RequestMethod.DELETE, path = "/catalog/{catalog-name}/database/{database-name}/table/{table-name}", consumes = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.NO_CONTENT)
@ApiOperation(position = 4, value = "Remove the tags from the given table", notes = "Remove the tags from the given table")
@ApiResponses({ @ApiResponse(code = HttpURLConnection.HTTP_NO_CONTENT, message = "The tags were successfully deleted from the table"), @ApiResponse(code = HttpURLConnection.HTTP_NOT_FOUND, message = "The requested catalog or database or table cannot be located") })
public void removeTableTags(@ApiParam(value = "The name of the catalog", required = true) @PathVariable("catalog-name") final String catalogName, @ApiParam(value = "The name of the database", required = true) @PathVariable("database-name") final String databaseName, @ApiParam(value = "The name of the table", required = true) @PathVariable("table-name") final String tableName, @ApiParam(value = "True if all tags need to be removed") @RequestParam(name = "all", defaultValue = "false") final boolean deleteAll, @ApiParam(value = "Tags to be removed from the given table") @Nullable @RequestBody(required = false) final Set<String> tags) {
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final QualifiedName name = this.requestWrapper.qualifyName(() -> QualifiedName.ofTable(catalogName, databaseName, tableName));
    this.requestWrapper.processRequest(name, "TagV1Resource.removeTableTags", () -> {
        // TODO: Business logic in API tier...
        if (!this.tableService.exists(name)) {
            // Delete tags if exists
            this.tagService.delete(name, false);
            throw new TableNotFoundException(name);
        }
        final TableDto oldTable = this.tableService.get(name, GetTableServiceParameters.builder().includeInfo(true).includeDataMetadata(true).includeDefinitionMetadata(true).disableOnReadMetadataIntercetor(false).build()).orElseThrow(IllegalStateException::new);
        this.tagService.removeTableTags(name, deleteAll, tags, true);
        final TableDto currentTable = this.tableService.get(name, GetTableServiceParameters.builder().includeInfo(true).includeDataMetadata(true).includeDefinitionMetadata(true).disableOnReadMetadataIntercetor(false).build()).orElseThrow(IllegalStateException::new);
        this.eventBus.postAsync(new MetacatUpdateTablePostEvent(name, metacatRequestContext, this, oldTable, currentTable));
        return null;
    });
}