Examples with DatabaseDto com.netflix.metacat.common.dto.DatabaseDto used on opensource projects

Example 6 with DatabaseDto

use of com.netflix.metacat.common.dto.DatabaseDto in project metacat by Netflix.

the class ElasticSearchEventHandlers method metacatCreateDatabasePostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatCreateDatabasePostEventHandler(final MetacatCreateDatabasePostEvent event) {
    log.debug("Received CreateDatabaseEvent {}", event);
    this.databaseCreateEventsDelayTimer.record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS);
    this.databaseCreateTimer.record(() -> {
        final DatabaseDto dto = event.getDatabase();
        final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false);
        es.save(ElasticSearchDoc.Type.database.name(), doc.getId(), doc);
    });
}

Example 7 with DatabaseDto

use of com.netflix.metacat.common.dto.DatabaseDto in project metacat by Netflix.

the class ElasticSearchRefresh method _processDatabases.

/**
 * Process the list of databases.
 *
 * @param catalogName   catalog name
 * @param databaseNames database names
 * @return future
 */
@SuppressWarnings("checkstyle:methodname")
private ListenableFuture<Void> _processDatabases(final QualifiedName catalogName, final List<QualifiedName> databaseNames) {
    ListenableFuture<Void> resultFuture = null;
    log.info("Full refresh of catalog {} for databases({}): {}", catalogName, databaseNames.size(), databaseNames);
    final List<ListenableFuture<DatabaseDto>> getDatabaseFutures = databaseNames.stream().map(databaseName -> service.submit(() -> {
        DatabaseDto result = null;
        try {
            result = getDatabase(databaseName);
        } catch (Exception e) {
            log.error("Failed to retrieve database: {}", databaseName);
            elasticSearchUtil.log("ElasticSearchRefresh.getDatabase", ElasticSearchDoc.Type.database.name(), databaseName.toString(), null, e.getMessage(), e, true);
        }
        return result;
    })).collect(Collectors.toList());
    if (getDatabaseFutures != null && !getDatabaseFutures.isEmpty()) {
        resultFuture = Futures.transformAsync(Futures.successfulAsList(getDatabaseFutures), input -> {
            final ListenableFuture<Void> processDatabaseFuture = indexDatabaseDtos(catalogName, input);
            final List<ListenableFuture<Void>> processDatabaseFutures = input.stream().filter(NOT_NULL).map(databaseDto -> {
                final List<QualifiedName> tableNames = databaseDto.getTables().stream().map(s -> QualifiedName.ofTable(databaseDto.getName().getCatalogName(), databaseDto.getName().getDatabaseName(), s)).collect(Collectors.toList());
                log.info("Full refresh of database {} for tables({}): {}", databaseDto.getName(), databaseDto.getTables().size(), databaseDto.getTables());
                return processTables(databaseDto.getName(), tableNames);
            }).filter(NOT_NULL).collect(Collectors.toList());
            processDatabaseFutures.add(processDatabaseFuture);
            return Futures.transform(Futures.successfulAsList(processDatabaseFutures), Functions.constant(null));
        });
    }
    return resultFuture;
}

Example 8 with DatabaseDto

use of com.netflix.metacat.common.dto.DatabaseDto in project metacat by Netflix.

the class ElasticSearchRefresh method deleteUnmarkedEntities.

private void deleteUnmarkedEntities(final List<QualifiedName> qNames, final List<QualifiedName> excludeQualifiedNames) {
    log.info("Start: Delete unmarked entities");
    // 
    // get unmarked qualified names
    // check if it not exists
    // delete
    // 
    elasticSearchUtil.refresh();
    final MetacatRequestContext context = MetacatRequestContext.builder().userName("admin").clientAppName("metacat-refresh").apiUri("esRefresh").scheme("internal").build();
    final List<DatabaseDto> unmarkedDatabaseDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("database", qNames, refreshMarker, excludeQualifiedNames, DatabaseDto.class);
    if (!unmarkedDatabaseDtos.isEmpty()) {
        if (unmarkedDatabaseDtos.size() <= config.getElasticSearchThresholdUnmarkedDatabasesDelete()) {
            log.info("Start: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
            final List<String> unmarkedDatabaseNames = Lists.newArrayList();
            final List<DatabaseDto> deleteDatabaseDtos = unmarkedDatabaseDtos.stream().filter(databaseDto -> {
                boolean result = false;
                try {
                    unmarkedDatabaseNames.add(databaseDto.getName().toString());
                    final DatabaseDto dto = databaseService.get(databaseDto.getName(), GetDatabaseServiceParameters.builder().includeUserMetadata(false).includeTableNames(false).disableOnReadMetadataIntercetor(false).build());
                    if (dto == null) {
                        result = true;
                    }
                } catch (DatabaseNotFoundException ignored) {
                    result = true;
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked databases({}): {}", unmarkedDatabaseNames.size(), unmarkedDatabaseNames);
            log.info("Deleting databases({})", deleteDatabaseDtos.size());
            if (!deleteDatabaseDtos.isEmpty()) {
                final List<QualifiedName> deleteDatabaseQualifiedNames = deleteDatabaseDtos.stream().map(DatabaseDto::getName).collect(Collectors.toList());
                final List<String> deleteDatabaseNames = deleteDatabaseQualifiedNames.stream().map(QualifiedName::toString).collect(Collectors.toList());
                log.info("Deleting databases({}): {}", deleteDatabaseNames.size(), deleteDatabaseNames);
                userMetadataService.deleteDefinitionMetadata(deleteDatabaseQualifiedNames);
                elasticSearchUtil.softDelete("database", deleteDatabaseNames, context);
            }
            log.info("End: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
        } else {
            log.info("Count of unmarked databases({}) is more than the threshold {}", unmarkedDatabaseDtos.size(), config.getElasticSearchThresholdUnmarkedDatabasesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedDatabaseThreshholdReached.getMetricName())).increment();
        }
    }
    final List<TableDto> unmarkedTableDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("table", qNames, refreshMarker, excludeQualifiedNames, TableDto.class);
    if (!unmarkedTableDtos.isEmpty()) {
        if (unmarkedTableDtos.size() <= config.getElasticSearchThresholdUnmarkedTablesDelete()) {
            log.info("Start: Delete unmarked tables({})", unmarkedTableDtos.size());
            final List<String> unmarkedTableNames = Lists.newArrayList();
            final List<TableDto> deleteTableDtos = unmarkedTableDtos.stream().filter(tableDto -> {
                boolean result = false;
                try {
                    unmarkedTableNames.add(tableDto.getName().toString());
                    final Optional<TableDto> dto = tableService.get(tableDto.getName(), GetTableServiceParameters.builder().includeDataMetadata(false).disableOnReadMetadataIntercetor(false).includeInfo(true).includeDefinitionMetadata(false).build());
                    if (!dto.isPresent()) {
                        result = true;
                    }
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked tables({}): {}", unmarkedTableNames.size(), unmarkedTableNames);
            log.info("Deleting tables({})", deleteTableDtos.size());
            if (!deleteTableDtos.isEmpty()) {
                final List<String> deleteTableNames = deleteTableDtos.stream().map(dto -> dto.getName().toString()).collect(Collectors.toList());
                log.info("Deleting tables({}): {}", deleteTableNames.size(), deleteTableNames);
                userMetadataService.deleteMetadata("admin", Lists.newArrayList(deleteTableDtos));
                // Publish event. Elasticsearch event handler will take care of updating the index already
                // TODO: Re-evaluate events vs. direct calls for these types of situations like in Genie
                deleteTableDtos.forEach(tableDto -> {
                    tagService.delete(tableDto.getName(), false);
                    this.eventBus.postAsync(new MetacatDeleteTablePostEvent(tableDto.getName(), context, this, tableDto));
                });
            }
            log.info("End: Delete unmarked tables({})", unmarkedTableDtos.size());
        } else {
            log.info("Count of unmarked tables({}) is more than the threshold {}", unmarkedTableDtos.size(), config.getElasticSearchThresholdUnmarkedTablesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedTableThreshholdReached.getMetricName())).increment();
        }
    }
    log.info("End: Delete unmarked entities");
}

Example 9 with DatabaseDto

use of com.netflix.metacat.common.dto.DatabaseDto in project metacat by Netflix.

the class MetacatController method createDatabase.

/**
 * Creates the given database in the given catalog.
 *
 * @param catalogName              catalog name
 * @param databaseName             database name
 * @param databaseCreateRequestDto database create request
 */
@RequestMapping(method = RequestMethod.POST, path = "/catalog/{catalog-name}/database/{database-name}", consumes = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.CREATED)
@ApiOperation(position = 2, value = "Creates the given database in the given catalog", notes = "Given a catalog and a database name, creates the database in the catalog")
@ApiResponses({ @ApiResponse(code = HttpURLConnection.HTTP_CREATED, message = "The database was created"), @ApiResponse(code = HttpURLConnection.HTTP_NOT_FOUND, message = "The requested catalog or database cannot be located") })
@Override
public void createDatabase(@ApiParam(value = "The name of the catalog", required = true) @PathVariable("catalog-name") final String catalogName, @ApiParam(value = "The name of the database", required = true) @PathVariable("database-name") final String databaseName, @ApiParam(value = "The database information") @Nullable @RequestBody(required = false) final DatabaseCreateRequestDto databaseCreateRequestDto) {
    final QualifiedName name = this.requestWrapper.qualifyName(() -> QualifiedName.ofDatabase(catalogName, databaseName));
    this.requestWrapper.processRequest(name, "createDatabase", () -> {
        final DatabaseDto newDto = new DatabaseDto();
        newDto.setName(name);
        if (databaseCreateRequestDto != null) {
            newDto.setDefinitionMetadata(databaseCreateRequestDto.getDefinitionMetadata());
        }
        this.databaseService.create(name, newDto);
        return null;
    });
}

Example 10 with DatabaseDto

use of com.netflix.metacat.common.dto.DatabaseDto in project metacat by Netflix.

the class MetacatServiceHelper method postPreDeleteEvent.

/**
 * Calls the right method of the event bus for the given qualified name.
 *
 * @param name                  name
 * @param metacatRequestContext context
 */
public void postPreDeleteEvent(final QualifiedName name, final MetacatRequestContext metacatRequestContext) {
    if (name.isPartitionDefinition()) {
        final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto();
        partitionsSaveRequestDto.setPartitionIdsForDeletes(Lists.newArrayList(name.getPartitionName()));
        this.eventBus.postSync(new MetacatDeleteTablePartitionPreEvent(name, metacatRequestContext, this, partitionsSaveRequestDto));
    } else if (name.isViewDefinition()) {
        this.eventBus.postSync(new MetacatDeleteMViewPreEvent(name, metacatRequestContext, this));
    } else if (name.isTableDefinition()) {
        this.eventBus.postSync(new MetacatDeleteTablePreEvent(name, metacatRequestContext, this));
    } else if (name.isDatabaseDefinition()) {
        final DatabaseDto dto = new DatabaseDto();
        dto.setName(name);
        eventBus.postSync(new MetacatDeleteDatabasePreEvent(name, metacatRequestContext, this, dto));
    } else {
        throw new IllegalArgumentException(String.format("Invalid name %s", name));
    }
}