Examples with SemiTransactionalHiveMetastore com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore used on opensource projects

Example 11 with SemiTransactionalHiveMetastore

use of com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore in project presto by prestodb.

the class SqlStandardAccessControl method isDatabaseOwner.

private boolean isDatabaseOwner(ConnectorTransactionHandle transaction, ConnectorIdentity identity, MetastoreContext metastoreContext, String databaseName) {
    // all users are "owners" of the default database
    if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(databaseName)) {
        return true;
    }
    if (isAdmin(transaction, identity, metastoreContext)) {
        return true;
    }
    SemiTransactionalHiveMetastore metastore = getMetastore(transaction);
    Optional<Database> databaseMetadata = metastore.getDatabase(metastoreContext, databaseName);
    if (!databaseMetadata.isPresent()) {
        return false;
    }
    Database database = databaseMetadata.get();
    // a database can be owned by a user or role
    if (database.getOwnerType() == USER && identity.getUser().equals(database.getOwnerName())) {
        return true;
    }
    if (database.getOwnerType() == ROLE && isRoleEnabled(identity, (PrestoPrincipal p) -> metastore.listRoleGrants(metastoreContext, p), database.getOwnerName())) {
        return true;
    }
    return false;
}

Example 12 with SemiTransactionalHiveMetastore

use of com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore in project presto by prestodb.

the class SqlStandardAccessControl method hasAdminOptionForRoles.

private boolean hasAdminOptionForRoles(ConnectorTransactionHandle transaction, ConnectorIdentity identity, MetastoreContext metastoreContext, Set<String> roles) {
    if (isAdmin(transaction, identity, metastoreContext)) {
        return true;
    }
    SemiTransactionalHiveMetastore metastore = getMetastore(transaction);
    Set<String> rolesWithGrantOption = listApplicableRoles(new PrestoPrincipal(USER, identity.getUser()), (PrestoPrincipal p) -> metastore.listRoleGrants(metastoreContext, p)).filter(RoleGrant::isGrantable).map(RoleGrant::getRoleName).collect(toSet());
    return rolesWithGrantOption.containsAll(roles);
}

Example 13 with SemiTransactionalHiveMetastore

use of com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method listAllDataPaths.

public static List<String> listAllDataPaths(SemiTransactionalHiveMetastore metastore, String schemaName, String tableName) {
    ImmutableList.Builder<String> locations = ImmutableList.builder();
    Table table = metastore.getTable(schemaName, tableName).get();
    if (table.getStorage().getLocation() != null) {
        // For partitioned table, there should be nothing directly under this directory.
        // But including this location in the set makes the directory content assert more
        // extensive, which is desirable.
        locations.add(table.getStorage().getLocation());
    }
    Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName);
    if (partitionNames.isPresent()) {
        metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).values().stream().map(Optional::get).map(partition -> partition.getStorage().getLocation()).filter(location -> !location.startsWith(table.getStorage().getLocation())).forEach(locations::add);
    }
    return locations.build();
}

Example 14 with SemiTransactionalHiveMetastore

use of com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore in project presto by prestodb.

the class HiveFilterPushdown method pushdownFilter.

@VisibleForTesting
public static ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, ConnectorMetadata metadata, SemiTransactionalHiveMetastore metastore, RowExpressionService rowExpressionService, StandardFunctionResolution functionResolution, HivePartitionManager partitionManager, FunctionMetadataManager functionMetadataManager, ConnectorTableHandle tableHandle, RowExpression filter, Optional<ConnectorTableLayoutHandle> currentLayoutHandle) {
    checkArgument(!FALSE_CONSTANT.equals(filter), "Cannot pushdown filter that is always false");
    if (TRUE_CONSTANT.equals(filter) && currentLayoutHandle.isPresent()) {
        return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT);
    }
    // Split the filter into 3 groups of conjuncts:
    // - range filters that apply to entire columns,
    // - range filters that apply to subfields,
    // - the rest. Intersect these with possibly pre-existing filters.
    DomainTranslator.ExtractionResult<Subfield> decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor());
    if (currentLayoutHandle.isPresent()) {
        HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get();
        decomposedFilter = intersectExtractionResult(new DomainTranslator.ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate()), decomposedFilter);
    }
    if (decomposedFilter.getTupleDomain().isNone()) {
        return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
    }
    RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer().optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session);
    if (optimizedRemainingExpression instanceof ConstantExpression) {
        ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression;
        if (FALSE_CONSTANT.equals(constantExpression) || constantExpression.getValue() == null) {
            return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
        }
    }
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    TupleDomain<ColumnHandle> entireColumnDomain = decomposedFilter.getTupleDomain().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(columnHandles::get);
    if (currentLayoutHandle.isPresent()) {
        entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate());
    }
    Constraint<ColumnHandle> constraint = new Constraint<>(entireColumnDomain);
    // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate
    if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) {
        LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
        RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression());
        if (!TRUE_CONSTANT.equals(deterministicPredicate)) {
            ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate);
            constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate);
        }
    }
    HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    TupleDomain<Subfield> domainPredicate = withColumnDomains(ImmutableMap.<Subfield, Domain>builder().putAll(hivePartitionResult.getUnenforcedConstraint().transform(HiveFilterPushdown::toSubfield).getDomains().orElse(ImmutableMap.of())).putAll(decomposedFilter.getTupleDomain().transform(subfield -> !isEntireColumn(subfield) ? subfield : null).getDomains().orElse(ImmutableMap.of())).build());
    Set<String> predicateColumnNames = new HashSet<>();
    domainPredicate.getDomains().get().keySet().stream().map(Subfield::getRootName).forEach(predicateColumnNames::add);
    // Include only columns referenced in the optimized expression. Although the expression is sent to the worker node
    // unoptimized, the worker is expected to optimize the expression before executing.
    extractAll(optimizedRemainingExpression).stream().map(VariableReferenceExpression::getName).forEach(predicateColumnNames::add);
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
    List<RowExpression> conjuncts = extractConjuncts(decomposedFilter.getRemainingExpression());
    RowExpression dynamicFilterExpression = extractDynamicConjuncts(conjuncts, logicalRowExpressions);
    RowExpression remainingExpression = extractStaticConjuncts(conjuncts, logicalRowExpressions);
    remainingExpression = removeNestedDynamicFilters(remainingExpression);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, new HiveTableLayoutHandle(tableName, table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, remainingExpression, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), true, createTableLayoutString(session, rowExpressionService, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), remainingExpression, domainPredicate), currentLayoutHandle.map(layout -> ((HiveTableLayoutHandle) layout).getRequestedColumns()).orElse(Optional.empty()), false)), dynamicFilterExpression);
}

Example 15 with SemiTransactionalHiveMetastore

use of com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore in project presto by prestodb.

the class ThriftMetastoreUtil method listApplicableTablePrivileges.

public static Stream<HivePrivilegeInfo> listApplicableTablePrivileges(SemiTransactionalHiveMetastore metastore, ConnectorIdentity identity, MetastoreContext metastoreContext, String databaseName, String tableName, String user) {
    PrestoPrincipal userPrincipal = new PrestoPrincipal(USER, user);
    Stream<PrestoPrincipal> principals = Stream.concat(Stream.of(userPrincipal), listApplicableRoles(metastore, identity, userPrincipal, metastoreContext).map(role -> new PrestoPrincipal(ROLE, role)));
    return listTablePrivileges(identity, metastoreContext, metastore, databaseName, tableName, principals);
}