Examples with InvalidMetaException com.netflix.metacat.common.server.connectors.exception.InvalidMetaException used on opensource projects

Example 16 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class IcebergTableHandler method getIcebergTable.

/**
 * get iceberg table.
 *
 * @param tableName             table name
 * @param tableMetadataLocation table metadata location
 * @param includeInfoDetails    if true, will include more details like the manifest file content
 * @return iceberg table
 */
public IcebergTableWrapper getIcebergTable(final QualifiedName tableName, final String tableMetadataLocation, final boolean includeInfoDetails) {
    final long start = this.registry.clock().wallTime();
    try {
        this.icebergTableCriteria.checkCriteria(tableName, tableMetadataLocation);
        log.debug("Loading icebergTable {} from {}", tableName, tableMetadataLocation);
        final IcebergMetastoreTables icebergMetastoreTables = new IcebergMetastoreTables(new IcebergTableOps(conf, tableMetadataLocation, connectorContext.getConfig(), icebergTableOpsProxy));
        final Table table = icebergMetastoreTables.loadTable(HiveTableUtil.qualifiedNameToTableIdentifier(tableName));
        final Map<String, String> extraProperties = Maps.newHashMap();
        if (includeInfoDetails) {
            extraProperties.put(DirectSqlTable.PARAM_METADATA_CONTENT, TableMetadataParser.toJson(icebergMetastoreTables.getTableOps().current()));
        }
        return new IcebergTableWrapper(table, extraProperties);
    } catch (NotFoundException | NoSuchTableException e) {
        throw new InvalidMetaException(tableName, e);
    } finally {
        final long duration = registry.clock().wallTime() - start;
        log.info("Time taken to getIcebergTable {} is {} ms", tableName, duration);
        this.recordTimer(IcebergRequestMetrics.TagLoadTable.getMetricName(), duration);
        this.increaseCounter(IcebergRequestMetrics.TagLoadTable.getMetricName(), tableName);
    }
}

Example 17 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class MetacatHiveClient method addDropPartitions.

/**
 * {@inheritDoc}.
 */
@Override
public void addDropPartitions(final String dbName, final String tableName, final List<Partition> partitions, final List<String> delPartitionNames) throws TException {
    try (HiveMetastoreClient client = createMetastoreClient()) {
        try {
            dropHivePartitions(client, dbName, tableName, delPartitionNames);
            client.add_partitions(partitions);
        } catch (MetaException | InvalidObjectException e) {
            throw new InvalidMetaException("One or more partitions are invalid.", e);
        } catch (TException e) {
            throw new TException(String.format("Internal server error adding/dropping partitions for table %s.%s", dbName, tableName), e);
        }
    }
}

Example 18 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorPartitionService method getPartitions.

/**
     * {@inheritDoc}.
     */
@Override
public List<PartitionInfo> getPartitions(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName, @Nonnull @NonNull final PartitionListRequest partitionsRequest) {
    try {
        final List<Partition> partitions = getPartitions(tableName, partitionsRequest.getFilter(), partitionsRequest.getPartitionNames(), partitionsRequest.getSort(), partitionsRequest.getPageable());
        final Table table = metacatHiveClient.getTableByName(tableName.getDatabaseName(), tableName.getTableName());
        final TableInfo tableInfo = hiveMetacatConverters.toTableInfo(tableName, table);
        final List<PartitionInfo> partitionInfos = new ArrayList<>();
        for (Partition partition : partitions) {
            partitionInfos.add(hiveMetacatConverters.toPartitionInfo(tableInfo, partition));
        }
        return partitionInfos;
    } catch (NoSuchObjectException exception) {
        throw new TableNotFoundException(tableName, exception);
    } catch (MetaException | InvalidObjectException e) {
        throw new InvalidMetaException("Invalid metadata for " + tableName, e);
    } catch (TException e) {
        throw new ConnectorException(String.format("Failed get partitions for hive table %s", tableName), e);
    }
}

Example 19 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorPartitionService method savePartitions.

/**
     * {@inheritDoc}.
     */
@Override
public PartitionsSaveResponse savePartitions(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName tableName, @Nonnull @NonNull final PartitionsSaveRequest partitionsSaveRequest) {
    final String databasename = tableName.getDatabaseName();
    final String tablename = tableName.getTableName();
    // New partitions
    final List<Partition> hivePartitions = Lists.newArrayList();
    try {
        final Table table = metacatHiveClient.getTableByName(databasename, tablename);
        final List<PartitionInfo> partitionInfos = partitionsSaveRequest.getPartitions();
        // New partition ids
        final List<String> addedPartitionIds = Lists.newArrayList();
        // Updated partition ids
        final List<String> existingPartitionIds = Lists.newArrayList();
        // Existing partitions
        final List<Partition> existingHivePartitions = Lists.newArrayList();
        // Existing partition map
        Map<String, Partition> existingPartitionMap = Collections.emptyMap();
        if (partitionsSaveRequest.getCheckIfExists()) {
            final List<String> partitionNames = partitionInfos.stream().map(partition -> {
                final String partitionName = partition.getName().getPartitionName();
                PartitionUtil.validatePartitionName(partitionName, getPartitionKeys(table.getPartitionKeys()));
                return partitionName;
            }).collect(Collectors.toList());
            existingPartitionMap = getPartitionsByNames(table, partitionNames);
        }
        final TableInfo tableInfo = hiveMetacatConverters.toTableInfo(tableName, table);
        for (PartitionInfo partitionInfo : partitionInfos) {
            final String partitionName = partitionInfo.getName().getPartitionName();
            final Partition hivePartition = existingPartitionMap.get(partitionName);
            if (hivePartition == null) {
                addedPartitionIds.add(partitionName);
                hivePartitions.add(hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo));
            } else {
                //unless we alterifExists
                if (partitionsSaveRequest.getAlterIfExists()) {
                    final Partition existingPartition = hiveMetacatConverters.fromPartitionInfo(tableInfo, partitionInfo);
                    existingPartitionIds.add(partitionName);
                    existingPartition.setParameters(hivePartition.getParameters());
                    existingPartition.setCreateTime(hivePartition.getCreateTime());
                    existingPartition.setLastAccessTime(hivePartition.getLastAccessTime());
                    existingHivePartitions.add(existingPartition);
                }
            }
        }
        final Set<String> deletePartitionIds = Sets.newHashSet();
        if (!partitionsSaveRequest.getAlterIfExists()) {
            deletePartitionIds.addAll(existingPartitionIds);
        }
        if (partitionsSaveRequest.getPartitionIdsForDeletes() != null) {
            deletePartitionIds.addAll(partitionsSaveRequest.getPartitionIdsForDeletes());
        }
        if (partitionsSaveRequest.getAlterIfExists() && !existingHivePartitions.isEmpty()) {
            copyTableSdToPartitionSd(existingHivePartitions, table);
            metacatHiveClient.alterPartitions(databasename, tablename, existingHivePartitions);
        }
        copyTableSdToPartitionSd(hivePartitions, table);
        metacatHiveClient.addDropPartitions(databasename, tablename, hivePartitions, Lists.newArrayList(deletePartitionIds));
        final PartitionsSaveResponse result = new PartitionsSaveResponse();
        result.setAdded(addedPartitionIds);
        result.setUpdated(existingPartitionIds);
        return result;
    } catch (NoSuchObjectException exception) {
        if (exception.getMessage() != null && exception.getMessage().startsWith("Partition doesn't exist")) {
            throw new PartitionNotFoundException(tableName, "", exception);
        } else {
            throw new TableNotFoundException(tableName, exception);
        }
    } catch (MetaException | InvalidObjectException exception) {
        throw new InvalidMetaException("One or more partitions are invalid.", exception);
    } catch (AlreadyExistsException e) {
        final List<String> ids = getFakePartitionName(hivePartitions);
        throw new PartitionAlreadyExistsException(tableName, ids, e);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed savePartitions hive table %s", tableName), exception);
    }
}

Example 20 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorTableService method listNames.

/**
     * {@inheritDoc}.
     */
@Override
public List<QualifiedName> listNames(@Nonnull @NonNull final ConnectorContext requestContext, @Nonnull @NonNull final QualifiedName name, @Nullable final QualifiedName prefix, @Nullable final Sort sort, @Nullable final Pageable pageable) {
    try {
        final List<QualifiedName> qualifiedNames = Lists.newArrayList();
        final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : null;
        for (String tableName : metacatHiveClient.getAllTables(name.getDatabaseName())) {
            if (tableFilter == null || tableName.startsWith(tableFilter)) {
                final QualifiedName qualifiedName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), tableName);
                if (prefix != null && !qualifiedName.toString().startsWith(prefix.toString())) {
                    continue;
                }
                qualifiedNames.add(qualifiedName);
            }
        }
        ////supporting sort by qualified name only
        if (sort != null) {
            ConnectorUtils.sort(qualifiedNames, sort, Comparator.comparing(QualifiedName::toString));
        }
        return ConnectorUtils.paginate(qualifiedNames, pageable);
    } catch (MetaException exception) {
        throw new InvalidMetaException(name, exception);
    } catch (NoSuchObjectException exception) {
        throw new DatabaseNotFoundException(name, exception);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed listNames hive table %s", name), exception);
    }
}