Examples with TableDto com.netflix.metacat.common.dto.TableDto used on opensource projects

Example 6 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class ElasticSearchEventHandlers method metacatRenameTablePostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatRenameTablePostEventHandler(final MetacatRenameTablePostEvent event) {
    log.debug("Received RenameTableEvent {}", event);
    this.tableRenameEventsDelayTimer.record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS);
    this.tableRenameTimer.record(() -> {
        es.delete(ElasticSearchDoc.Type.table.name(), event.getName().toString());
        final TableDto dto = event.getCurrentTable();
        final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false);
        es.save(ElasticSearchDoc.Type.table.name(), doc.getId(), doc);
    });
}

Example 7 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class ElasticSearchEventHandlers method metacatDeleteTablePostEventHandler.

/**
 * Subscriber.
 *
 * @param event event
 */
@EventListener
public void metacatDeleteTablePostEventHandler(final MetacatDeleteTablePostEvent event) {
    log.debug("Received DeleteTableEvent {}", event);
    this.tableDeleteEventsDelayTimer.record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS);
    this.tableDeleteTimer.record(() -> {
        final TableDto dto = event.getTable();
        es.softDelete(ElasticSearchDoc.Type.table.name(), dto.getName().toString(), event.getRequestContext());
        if (config.isElasticSearchPublishPartitionEnabled()) {
            try {
                final List<String> partitionIdsToBeDeleted = es.getIdsByQualifiedName(ElasticSearchDoc.Type.partition.name(), dto.getName());
                es.delete(ElasticSearchDoc.Type.partition.name(), partitionIdsToBeDeleted);
            } catch (Exception e) {
                log.warn("Failed deleting the partitions for the dropped table/view:{}", dto.getName());
            }
        }
    });
}

Example 8 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class ElasticSearchRefresh method deleteUnmarkedEntities.

private void deleteUnmarkedEntities(final List<QualifiedName> qNames, final List<QualifiedName> excludeQualifiedNames) {
    log.info("Start: Delete unmarked entities");
    // 
    // get unmarked qualified names
    // check if it not exists
    // delete
    // 
    elasticSearchUtil.refresh();
    final MetacatRequestContext context = MetacatRequestContext.builder().userName("admin").clientAppName("metacat-refresh").apiUri("esRefresh").scheme("internal").build();
    final List<DatabaseDto> unmarkedDatabaseDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("database", qNames, refreshMarker, excludeQualifiedNames, DatabaseDto.class);
    if (!unmarkedDatabaseDtos.isEmpty()) {
        if (unmarkedDatabaseDtos.size() <= config.getElasticSearchThresholdUnmarkedDatabasesDelete()) {
            log.info("Start: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
            final List<String> unmarkedDatabaseNames = Lists.newArrayList();
            final List<DatabaseDto> deleteDatabaseDtos = unmarkedDatabaseDtos.stream().filter(databaseDto -> {
                boolean result = false;
                try {
                    unmarkedDatabaseNames.add(databaseDto.getName().toString());
                    final DatabaseDto dto = databaseService.get(databaseDto.getName(), GetDatabaseServiceParameters.builder().includeUserMetadata(false).includeTableNames(false).disableOnReadMetadataIntercetor(false).build());
                    if (dto == null) {
                        result = true;
                    }
                } catch (DatabaseNotFoundException ignored) {
                    result = true;
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked databases({}): {}", unmarkedDatabaseNames.size(), unmarkedDatabaseNames);
            log.info("Deleting databases({})", deleteDatabaseDtos.size());
            if (!deleteDatabaseDtos.isEmpty()) {
                final List<QualifiedName> deleteDatabaseQualifiedNames = deleteDatabaseDtos.stream().map(DatabaseDto::getName).collect(Collectors.toList());
                final List<String> deleteDatabaseNames = deleteDatabaseQualifiedNames.stream().map(QualifiedName::toString).collect(Collectors.toList());
                log.info("Deleting databases({}): {}", deleteDatabaseNames.size(), deleteDatabaseNames);
                userMetadataService.deleteDefinitionMetadata(deleteDatabaseQualifiedNames);
                elasticSearchUtil.softDelete("database", deleteDatabaseNames, context);
            }
            log.info("End: Delete unmarked databases({})", unmarkedDatabaseDtos.size());
        } else {
            log.info("Count of unmarked databases({}) is more than the threshold {}", unmarkedDatabaseDtos.size(), config.getElasticSearchThresholdUnmarkedDatabasesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedDatabaseThreshholdReached.getMetricName())).increment();
        }
    }
    final List<TableDto> unmarkedTableDtos = elasticSearchUtil.getQualifiedNamesByMarkerByNames("table", qNames, refreshMarker, excludeQualifiedNames, TableDto.class);
    if (!unmarkedTableDtos.isEmpty()) {
        if (unmarkedTableDtos.size() <= config.getElasticSearchThresholdUnmarkedTablesDelete()) {
            log.info("Start: Delete unmarked tables({})", unmarkedTableDtos.size());
            final List<String> unmarkedTableNames = Lists.newArrayList();
            final List<TableDto> deleteTableDtos = unmarkedTableDtos.stream().filter(tableDto -> {
                boolean result = false;
                try {
                    unmarkedTableNames.add(tableDto.getName().toString());
                    final Optional<TableDto> dto = tableService.get(tableDto.getName(), GetTableServiceParameters.builder().includeDataMetadata(false).disableOnReadMetadataIntercetor(false).includeInfo(true).includeDefinitionMetadata(false).build());
                    if (!dto.isPresent()) {
                        result = true;
                    }
                } catch (Exception ignored) {
                }
                return result;
            }).collect(Collectors.toList());
            log.info("Unmarked tables({}): {}", unmarkedTableNames.size(), unmarkedTableNames);
            log.info("Deleting tables({})", deleteTableDtos.size());
            if (!deleteTableDtos.isEmpty()) {
                final List<String> deleteTableNames = deleteTableDtos.stream().map(dto -> dto.getName().toString()).collect(Collectors.toList());
                log.info("Deleting tables({}): {}", deleteTableNames.size(), deleteTableNames);
                userMetadataService.deleteMetadata("admin", Lists.newArrayList(deleteTableDtos));
                // Publish event. Elasticsearch event handler will take care of updating the index already
                // TODO: Re-evaluate events vs. direct calls for these types of situations like in Genie
                deleteTableDtos.forEach(tableDto -> {
                    tagService.delete(tableDto.getName(), false);
                    this.eventBus.postAsync(new MetacatDeleteTablePostEvent(tableDto.getName(), context, this, tableDto));
                });
            }
            log.info("End: Delete unmarked tables({})", unmarkedTableDtos.size());
        } else {
            log.info("Count of unmarked tables({}) is more than the threshold {}", unmarkedTableDtos.size(), config.getElasticSearchThresholdUnmarkedTablesDelete());
            registry.counter(registry.createId(Metrics.CounterElasticSearchUnmarkedTableThreshholdReached.getMetricName())).increment();
        }
    }
    log.info("End: Delete unmarked entities");
}

Example 9 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class ElasticSearchUtilImpl method simpleSearch.

/**
 * {@inheritDoc}
 */
@Override
public List<TableDto> simpleSearch(final String searchString) {
    final List<TableDto> result = Lists.newArrayList();
    final SearchResponse response = client.prepareSearch(esIndex).setTypes(ElasticSearchDoc.Type.table.name()).setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(QueryBuilders.termQuery("_all", searchString)).setSize(Integer.MAX_VALUE).execute().actionGet();
    if (response.getHits().getHits().length != 0) {
        result.addAll(parseResponse(response, TableDto.class));
    }
    return result;
}

Example 10 with TableDto

use of com.netflix.metacat.common.dto.TableDto in project metacat by Netflix.

the class CatalogThriftHiveMetastore method alter_table_with_environment_context.

/**
 * {@inheritDoc}
 */
@Override
public void alter_table_with_environment_context(final String dbname, final String tblName, final Table newTbl, @Nullable final EnvironmentContext environmentContext) throws TException {
    requestWrapper("alter_table_with_environment_context", new Object[] { dbname, tblName, newTbl, environmentContext }, () -> {
        final String databaseName = normalizeIdentifier(dbname);
        final String tableName = normalizeIdentifier(tblName);
        final QualifiedName oldName = QualifiedName.ofTable(catalogName, databaseName, tableName);
        final QualifiedName newName = QualifiedName.ofTable(catalogName, newTbl.getDbName(), newTbl.getTableName());
        final TableDto dto = hiveConverters.hiveToMetacatTable(newName, newTbl);
        if (!oldName.equals(newName)) {
            v1.renameTable(catalogName, oldName.getDatabaseName(), oldName.getTableName(), newName.getTableName());
        }
        v1.updateTable(catalogName, dbname, newName.getTableName(), dto);
        return null;
    });
}