Examples with QualifiedName com.netflix.metacat.common.QualifiedName used on opensource projects

Example 66 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class PartitionServiceImpl method list.

@Override
public List<PartitionDto> list(final QualifiedName name, final String filter, final List<String> partitionNames, final Sort sort, final Pageable pageable, final boolean includeUserDefinitionMetadata, final boolean includeUserDataMetadata, final boolean includePartitionDetails) {
    if (Strings.isNullOrEmpty(filter) && (pageable == null || !pageable.isPageable()) && (partitionNames == null || partitionNames.isEmpty()) && config.getQualifiedNamesToThrowErrorWhenNoFilterOnListPartitions().contains(name)) {
        throw new IllegalArgumentException(String.format("No filter or limit specified for table %s", name));
    }
    final MetacatRequestContext metacatRequestContext = MetacatContextManager.getContext();
    final ConnectorPartitionService service = connectorManager.getPartitionService(name.getCatalogName());
    final GetPartitionsRequestDto requestDto = new GetPartitionsRequestDto();
    requestDto.setFilter(filter);
    requestDto.setIncludePartitionDetails(includePartitionDetails);
    requestDto.setPartitionNames(partitionNames);
    final ConnectorContext connectorContext = converterUtil.toConnectorContext(metacatRequestContext);
    final List<PartitionInfo> resultInfo = service.getPartitions(connectorContext, name, converterUtil.toPartitionListRequest(requestDto, pageable, sort));
    List<PartitionDto> result = Lists.newArrayList();
    if (resultInfo != null && !resultInfo.isEmpty()) {
        result = resultInfo.stream().map(converterUtil::toPartitionDto).collect(Collectors.toList());
        final List<QualifiedName> names = Lists.newArrayList();
        final List<String> uris = Lists.newArrayList();
        result.forEach(partitionDto -> {
            names.add(partitionDto.getName());
            uris.add(partitionDto.getDataUri());
        });
        registry.gauge(this.partitionGetCountId.withTags(new HashMap<>(name.parts())), result.size());
        log.info("Got {} partitions for {} using filter: {} and partition names: {}", result.size(), name, filter, partitionNames);
        if (includeUserDefinitionMetadata || includeUserDataMetadata) {
            final List<ListenableFuture<Map<String, ObjectNode>>> futures = Lists.newArrayList();
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDefinitionMetadata ? userMetadataService.getDefinitionMetadataMap(names) : Maps.newHashMap()));
            futures.add(threadServiceManager.getExecutor().submit(() -> includeUserDataMetadata ? userMetadataService.getDataMetadataMap(uris) : Maps.newHashMap()));
            try {
                final List<Map<String, ObjectNode>> metadataResults = Futures.successfulAsList(futures).get(1, TimeUnit.HOURS);
                final Map<String, ObjectNode> definitionMetadataMap = metadataResults.get(0);
                final Map<String, ObjectNode> dataMetadataMap = metadataResults.get(1);
                result.forEach(partitionDto -> userMetadataService.populateMetadata(partitionDto, definitionMetadataMap.get(partitionDto.getName().toString()), dataMetadataMap.get(partitionDto.getDataUri())));
            } catch (Exception e) {
                Throwables.propagate(e);
            }
        }
    }
    return result;
}

Example 67 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class PartitionServiceImpl method delete.

@Override
public void delete(@Nonnull final QualifiedName name) {
    final QualifiedName tableName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), name.getTableName());
    delete(tableName, Lists.newArrayList(name.getPartitionName()));
}

Example 68 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class PartitionServiceImpl method get.

@Override
public PartitionDto get(@Nonnull final QualifiedName name) {
    PartitionDto result = null;
    final QualifiedName tableName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), name.getTableName());
    final List<PartitionDto> dtos = list(tableName, null, Lists.newArrayList(name.getPartitionName()), null, null, true, true, true);
    if (!dtos.isEmpty()) {
        result = dtos.get(0);
    }
    return result;
}

Example 69 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MysqlUserMetadataService method searchByOwners.

@Override
public List<QualifiedName> searchByOwners(final Set<String> owners) {
    final List<QualifiedName> result = Lists.newArrayList();
    final StringBuilder query = new StringBuilder(SQL.SEARCH_DEFINITION_METADATA_NAMES);
    final List<Object> paramList = Lists.newArrayList();
    query.append(" where 1=0");
    owners.forEach(s -> {
        query.append(" or data like ?");
        paramList.add("%\"userId\":\"" + s.trim() + "\"%");
    });
    final Object[] params = new Object[paramList.size()];
    final Connection connection = DBUtil.getReadConnection(poolingDataSource);
    try {
        // Handler for reading the result set
        final ResultSetHandler<Void> handler = rs -> {
            while (rs.next()) {
                final String definitionName = rs.getString("name");
                result.add(QualifiedName.fromString(definitionName, false));
            }
            return null;
        };
        new QueryRunner().query(connection, query.toString(), handler, paramList.toArray(params));
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to get definition data", e);
    } finally {
        DBUtil.closeReadConnection(connection);
    }
    return result;
}

Example 70 with QualifiedName

use of com.netflix.metacat.common.QualifiedName in project metacat by Netflix.

the class MysqlUserMetadataService method saveMetadatas.

@Override
public void saveMetadatas(final String user, final List<? extends HasMetadata> metadatas, final boolean merge) {
    try {
        final Connection conn = poolingDataSource.getConnection();
        try {
            @SuppressWarnings("unchecked") final List<List<HasMetadata>> subLists = Lists.partition((List<HasMetadata>) metadatas, config.getUserMetadataMaxInClauseItems());
            for (List<HasMetadata> hasMetadatas : subLists) {
                final List<String> uris = Lists.newArrayList();
                final List<QualifiedName> names = Lists.newArrayList();
                // Get the names and uris
                final List<HasDefinitionMetadata> definitionMetadatas = Lists.newArrayList();
                final List<HasDataMetadata> dataMetadatas = Lists.newArrayList();
                hasMetadatas.stream().forEach(hasMetadata -> {
                    if (hasMetadata instanceof HasDefinitionMetadata) {
                        final HasDefinitionMetadata oDef = (HasDefinitionMetadata) hasMetadata;
                        names.add(oDef.getDefinitionName());
                        if (oDef.getDefinitionMetadata() != null) {
                            definitionMetadatas.add(oDef);
                        }
                    }
                    if (hasMetadata instanceof HasDataMetadata) {
                        final HasDataMetadata oData = (HasDataMetadata) hasMetadata;
                        if (oData.isDataExternal() && oData.getDataMetadata() != null && oData.getDataMetadata().size() > 0) {
                            uris.add(oData.getDataUri());
                            dataMetadatas.add(oData);
                        }
                    }
                });
                if (!definitionMetadatas.isEmpty() || !dataMetadatas.isEmpty()) {
                    // Get the existing metadata based on the names and uris
                    final Map<String, ObjectNode> definitionMap = getDefinitionMetadataMap(names);
                    final Map<String, ObjectNode> dataMap = getDataMetadataMap(uris);
                    // Curate the list of existing and new metadatas
                    final List<Object[]> insertDefinitionMetadatas = Lists.newArrayList();
                    final List<Object[]> updateDefinitionMetadatas = Lists.newArrayList();
                    final List<Object[]> insertDataMetadatas = Lists.newArrayList();
                    final List<Object[]> updateDataMetadatas = Lists.newArrayList();
                    definitionMetadatas.stream().forEach(oDef -> {
                        final QualifiedName qualifiedName = oDef.getDefinitionName();
                        if (qualifiedName != null && oDef.getDefinitionMetadata() != null && oDef.getDefinitionMetadata().size() != 0) {
                            final String name = qualifiedName.toString();
                            final ObjectNode oNode = definitionMap.get(name);
                            if (oNode == null) {
                                insertDefinitionMetadatas.add(new Object[] { metacatJson.toJsonString(oDef.getDefinitionMetadata()), user, user, name });
                            } else {
                                metacatJson.mergeIntoPrimary(oNode, oDef.getDefinitionMetadata());
                                updateDefinitionMetadatas.add(new Object[] { metacatJson.toJsonString(oNode), user, name });
                            }
                        }
                    });
                    dataMetadatas.stream().forEach(oData -> {
                        final String uri = oData.getDataUri();
                        final ObjectNode oNode = dataMap.get(uri);
                        if (oData.getDataMetadata() != null && oData.getDataMetadata().size() != 0) {
                            if (oNode == null) {
                                insertDataMetadatas.add(new Object[] { metacatJson.toJsonString(oData.getDataMetadata()), user, user, uri });
                            } else {
                                metacatJson.mergeIntoPrimary(oNode, oData.getDataMetadata());
                                updateDataMetadatas.add(new Object[] { metacatJson.toJsonString(oNode), user, uri });
                            }
                        }
                    });
                    //Now run the queries
                    final QueryRunner runner = new QueryRunner();
                    if (!insertDefinitionMetadatas.isEmpty()) {
                        runner.batch(conn, SQL.INSERT_DEFINITION_METADATA, insertDefinitionMetadatas.toArray(new Object[insertDefinitionMetadatas.size()][4]));
                    }
                    if (!updateDefinitionMetadatas.isEmpty()) {
                        runner.batch(conn, SQL.UPDATE_DEFINITION_METADATA, updateDefinitionMetadatas.toArray(new Object[updateDefinitionMetadatas.size()][3]));
                    }
                    if (!insertDataMetadatas.isEmpty()) {
                        runner.batch(conn, SQL.INSERT_DATA_METADATA, insertDataMetadatas.toArray(new Object[insertDataMetadatas.size()][4]));
                    }
                    if (!updateDataMetadatas.isEmpty()) {
                        runner.batch(conn, SQL.UPDATE_DATA_METADATA, updateDataMetadatas.toArray(new Object[updateDataMetadatas.size()][3]));
                    }
                }
            }
            conn.commit();
        } catch (SQLException e) {
            conn.rollback();
            throw e;
        } finally {
            conn.close();
        }
    } catch (SQLException e) {
        log.error("Sql exception", e);
        throw new UserMetadataServiceException("Failed to save metadata", e);
    }
}