Examples with InvalidMetaException com.netflix.metacat.common.server.connectors.exception.InvalidMetaException used on opensource projects

Example 26 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class DirectSqlTable method updateIcebergTable.

/**
 * Locks and updates the iceberg table for update so that no other request can modify the table at the same time.
 * 1. Gets the table parameters and locks the requested records. If lock cannot be attained,
 * the request to update fails
 * 2. Validates the metadata location
 * 3. If validated, updates the table parameters.
 * @param tableInfo table info
 */
@Transactional(propagation = Propagation.REQUIRES_NEW)
public void updateIcebergTable(final TableInfo tableInfo) {
    final QualifiedName tableName = tableInfo.getName();
    final Map<String, String> newTableMetadata = tableInfo.getMetadata();
    // 
    if (newTableMetadata == null || newTableMetadata.isEmpty()) {
        final String message = String.format("No parameters defined for iceberg table %s", tableName);
        log.warn(message);
        throw new InvalidMetaException(tableName, message, null);
    }
    // 
    // If the previous metadata location is not empty, check if it is valid.
    // 
    final String previousMetadataLocation = newTableMetadata.get(PARAM_PREVIOUS_METADATA_LOCATION);
    if (config.isIcebergPreviousMetadataLocationCheckEnabled() && !StringUtils.isBlank(previousMetadataLocation)) {
        boolean doesPathExists = true;
        try {
            final Path previousMetadataPath = new Path(previousMetadataLocation);
            doesPathExists = warehouse.getFs(previousMetadataPath).exists(previousMetadataPath);
        } catch (Exception ignored) {
            log.warn(String.format("Failed getting the filesystem for %s", previousMetadataLocation));
            registry.counter(HiveMetrics.CounterFileSystemReadFailure.name()).increment();
        }
        if (!doesPathExists) {
            throw new InvalidMetaException(tableName, String.format("Invalid metadata for %s..Location %s does not exist", tableName, previousMetadataLocation), null);
        }
    }
    final Long tableId = getTableId(tableName);
    Map<String, String> existingTableMetadata = null;
    log.debug("Lock Iceberg table {}", tableName);
    try {
        existingTableMetadata = jdbcTemplate.query(SQL.TABLE_PARAMS_LOCK, new SqlParameterValue[] { new SqlParameterValue(Types.BIGINT, tableId) }, rs -> {
            final Map<String, String> result = Maps.newHashMap();
            while (rs.next()) {
                result.put(rs.getString(COL_PARAM_KEY), rs.getString(COL_PARAM_VALUE));
            }
            return result;
        });
    } catch (EmptyResultDataAccessException ex) {
        log.info(String.format("No parameters defined for iceberg table %s", tableName));
    } catch (Exception ex) {
        final String message = String.format("Failed getting a lock on iceberg table %s", tableName);
        log.warn(message, ex);
        throw new InvalidMetaException(tableName, message, null);
    }
    if (existingTableMetadata == null) {
        existingTableMetadata = Maps.newHashMap();
    }
    final boolean needUpdate = validateIcebergUpdate(tableName, existingTableMetadata, newTableMetadata);
    final String existingMetadataLocation = existingTableMetadata.get(PARAM_METADATA_LOCATION);
    final String newMetadataLocation = newTableMetadata.get(PARAM_METADATA_LOCATION);
    log.info("Servicing Iceberg commit request with tableId: {}, needUpdate: {}, " + "previousLocation: {}, existingLocation: {}, newLocation: {}", tableId, needUpdate, previousMetadataLocation, existingMetadataLocation, newMetadataLocation);
    if (needUpdate) {
        final MapDifference<String, String> diff = Maps.difference(existingTableMetadata, newTableMetadata);
        insertTableParams(tableId, diff.entriesOnlyOnRight());
        final Map<String, String> updateParams = diff.entriesDiffering().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, s -> s.getValue().rightValue()));
        updateTableParams(tableId, updateParams);
        // 
        // In addition to updating the table params, the table location in HMS needs to be updated for usage by
        // external tools, that access HMS directly
        // 
        updateTableLocation(tableId, tableInfo);
        log.info("Finished updating Iceberg table with tableId: {}", tableId);
    }
    log.debug("Unlocked Iceberg table {}", tableName);
}

Example 27 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorFastPartitionService method createLocationForPartition.

private void createLocationForPartition(final QualifiedName tableQName, final PartitionInfo partitionInfo, final Table table, final boolean doFileSystemCalls) {
    String location = partitionInfo.getSerde().getUri();
    Path path = null;
    if (StringUtils.isBlank(location)) {
        if (table.getSd() == null || table.getSd().getLocation() == null) {
            throw new InvalidMetaException(tableQName, null);
        }
        final String partitionName = partitionInfo.getName().getPartitionName();
        final List<String> partValues = PartitionUtil.getPartValuesFromPartName(tableQName, table, partitionName);
        final String escapedPartName = PartitionUtil.makePartName(table.getPartitionKeys(), partValues);
        path = new Path(table.getSd().getLocation(), escapedPartName);
    } else {
        try {
            path = warehouse.getDnsPath(new Path(location));
        } catch (Exception e) {
            throw new InvalidMetaException(String.format("Failed forming partition location; %s", location), e);
        }
    }
    if (path != null) {
        location = path.toString();
        partitionInfo.getSerde().setUri(location);
        if (doFileSystemCalls) {
            registry.counter(registry.createId(HiveMetrics.CounterHivePartitionFileSystemCall.getMetricName()).withTags(tableQName.parts())).increment();
            try {
                if (!warehouse.isDir(path)) {
                    // 
                    // Added to track the number of partition locations that do not exist before
                    // adding the partition metadata
                    registry.counter(registry.createId(HiveMetrics.CounterHivePartitionPathIsNotDir.getMetricName()).withTags(tableQName.parts())).increment();
                    log.info(String.format("Partition location %s does not exist for table %s", location, tableQName));
                    if (!warehouse.mkdirs(path, false)) {
                        throw new InvalidMetaException(String.format("%s is not a directory or unable to create one", location), null);
                    }
                }
            } catch (Exception e) {
                throw new InvalidMetaException(String.format("Failed creating partition location; %s", location), e);
            }
        }
    }
}

Example 28 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class PolarisConnectorTableService method update.

/**
 * {@inheritDoc}.
 */
@Override
public void update(final ConnectorRequestContext requestContext, final TableInfo tableInfo) {
    final QualifiedName name = tableInfo.getName();
    final Config conf = connectorContext.getConfig();
    try {
        final Map<String, String> newTableMetadata = tableInfo.getMetadata();
        if (MapUtils.isEmpty(newTableMetadata)) {
            final String message = String.format("No parameters defined for iceberg table %s", name);
            log.warn(message);
            throw new InvalidMetaException(name, message, null);
        }
        final String prevLoc = newTableMetadata.get(DirectSqlTable.PARAM_PREVIOUS_METADATA_LOCATION);
        final String newLoc = newTableMetadata.get(DirectSqlTable.PARAM_METADATA_LOCATION);
        if (StringUtils.isBlank(prevLoc) || StringUtils.isBlank(newLoc)) {
            final String message = String.format("Invalid metadata for %s. Provided previous %s or new %s location is empty.", name, prevLoc, newLoc);
            log.error(message);
            throw new InvalidMetaException(name, message, null);
        }
        if (conf.isIcebergPreviousMetadataLocationCheckEnabled() && !icebergTableHandler.doesMetadataLocationExist(name, prevLoc)) {
            final String message = String.format("Provided previous metadata location: %s for table: %s does not exist.", name, prevLoc);
            log.error(message);
            throw new InvalidMetaException(name, message, null);
        }
        // optimistically attempt to update metadata location
        final boolean updated = polarisStoreService.updateTableMetadataLocation(name.getDatabaseName(), name.getTableName(), prevLoc, newLoc);
        // if succeeded then done, else try to figure out why and throw corresponding exception
        if (updated) {
            requestContext.setIgnoreErrorsAfterUpdate(true);
            return;
        }
        final PolarisTableEntity table = polarisStoreService.getTable(name.getDatabaseName(), name.getTableName()).orElseThrow(() -> new TableNotFoundException(name));
        final String existingLoc = table.getMetadataLocation();
        if (StringUtils.isBlank(existingLoc)) {
            final String message = String.format("Invalid metadata location for %s existing location is empty.", name);
            log.error(message);
            throw new TablePreconditionFailedException(name, message, existingLoc, prevLoc);
        }
        if (StringUtils.equalsIgnoreCase(existingLoc, newLoc)) {
            log.warn("Existing metadata location is the same as new. Existing: {}, New: {}", existingLoc, newLoc);
            return;
        }
        if (!Objects.equals(existingLoc, prevLoc)) {
            final String message = String.format("Invalid metadata location for %s expected: %s, provided: %s", name, existingLoc, prevLoc);
            log.error(message);
            throw new TablePreconditionFailedException(name, message, existingLoc, prevLoc);
        }
    } catch (TableNotFoundException | InvalidMetaException | TablePreconditionFailedException exception) {
        throw exception;
    } catch (DataIntegrityViolationException exception) {
        throw new InvalidMetaException(name, exception);
    } catch (Exception exception) {
        final String msg = String.format("Failed updating polaris table %s", tableInfo.getName());
        log.error(msg, exception);
        throw new ConnectorException(msg, exception);
    }
}

Example 29 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorDatabaseService method list.

/**
 * {@inheritDoc}.
 */
@Override
public List<DatabaseInfo> list(final ConnectorRequestContext requestContext, final QualifiedName name, @Nullable final QualifiedName prefix, @Nullable final Sort sort, @Nullable final Pageable pageable) {
    try {
        final List<DatabaseInfo> databaseInfos = Lists.newArrayList();
        for (String databaseName : metacatHiveClient.getAllDatabases()) {
            final QualifiedName qualifiedName = QualifiedName.ofDatabase(name.getCatalogName(), databaseName);
            if (prefix != null && !qualifiedName.toString().startsWith(prefix.toString())) {
                continue;
            }
            databaseInfos.add(DatabaseInfo.builder().name(qualifiedName).build());
        }
        // supporting sort by name only
        if (sort != null) {
            ConnectorUtils.sort(databaseInfos, sort, Comparator.comparing(p -> p.getName().getDatabaseName()));
        }
        return ConnectorUtils.paginate(databaseInfos, pageable);
    } catch (MetaException exception) {
        throw new InvalidMetaException(name, exception);
    } catch (TException exception) {
        throw new ConnectorException(String.format("Failed list hive database %s", name), exception);
    }
}

Example 30 with InvalidMetaException

use of com.netflix.metacat.common.server.connectors.exception.InvalidMetaException in project metacat by Netflix.

the class HiveConnectorPartitionService method getPartitions.

private List<Partition> getPartitions(final QualifiedName tableName, @Nullable final String filter, @Nullable final List<String> partitionIds, @Nullable final Sort sort, @Nullable final Pageable pageable) {
    final String databasename = tableName.getDatabaseName();
    final String tablename = tableName.getTableName();
    try {
        final Table table = metacatHiveClient.getTableByName(databasename, tablename);
        List<Partition> partitionList = null;
        if (!Strings.isNullOrEmpty(filter)) {
            partitionList = metacatHiveClient.listPartitionsByFilter(databasename, tablename, filter);
        } else {
            if (partitionIds != null) {
                partitionList = metacatHiveClient.getPartitions(databasename, tablename, partitionIds);
            }
            if (partitionList == null || partitionList.isEmpty()) {
                partitionList = metacatHiveClient.getPartitions(databasename, tablename, null);
            }
        }
        final List<Partition> filteredPartitionList = Lists.newArrayList();
        partitionList.forEach(partition -> {
            final String partitionName = getNameOfPartition(table, partition);
            if (partitionIds == null || partitionIds.contains(partitionName)) {
                filteredPartitionList.add(partition);
            }
        });
        if (sort != null) {
            if (sort.getOrder() == SortOrder.DESC) {
                filteredPartitionList.sort(Collections.reverseOrder());
            } else {
                Collections.sort(filteredPartitionList);
            }
        }
        return ConnectorUtils.paginate(filteredPartitionList, pageable);
    } catch (NoSuchObjectException exception) {
        throw new TableNotFoundException(tableName, exception);
    } catch (MetaException | InvalidObjectException e) {
        throw new InvalidMetaException("Invalid metadata for " + tableName, e);
    } catch (TException e) {
        throw new ConnectorException(String.format("Failed get partitions for hive table %s", tableName), e);
    }
}