Search in sources :

Example 1 with INSERT

use of com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT in project presto by prestodb.

the class HiveTableOperations method commit.

@Override
public void commit(@Nullable TableMetadata base, TableMetadata metadata) {
    requireNonNull(metadata, "metadata is null");
    // if the metadata is already out of date, reject it
    if (!Objects.equals(base, current())) {
        throw new CommitFailedException("Cannot commit: stale table metadata for %s", getSchemaTableName());
    }
    // if the metadata is not changed, return early
    if (Objects.equals(base, metadata)) {
        return;
    }
    String newMetadataLocation = writeNewMetadata(metadata, version + 1);
    Table table;
    // getting a process-level lock per table to avoid concurrent commit attempts to the same table from the same
    // JVM process, which would result in unnecessary and costly HMS lock acquisition requests
    Optional<Long> lockId = Optional.empty();
    ReentrantLock tableLevelMutex = commitLockCache.getUnchecked(database + "." + tableName);
    tableLevelMutex.lock();
    try {
        try {
            lockId = Optional.of(metastore.lock(metastoreContext, database, tableName));
            if (base == null) {
                String tableComment = metadata.properties().get(TABLE_COMMENT);
                Map<String, String> parameters = new HashMap<>();
                parameters.put("EXTERNAL", "TRUE");
                parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE);
                parameters.put(METADATA_LOCATION, newMetadataLocation);
                if (tableComment != null) {
                    parameters.put(TABLE_COMMENT, tableComment);
                }
                Table.Builder builder = Table.builder().setDatabaseName(database).setTableName(tableName).setOwner(owner.orElseThrow(() -> new IllegalStateException("Owner not set"))).setTableType(PrestoTableType.EXTERNAL_TABLE).setDataColumns(toHiveColumns(metadata.schema().columns())).withStorage(storage -> storage.setLocation(metadata.location())).withStorage(storage -> storage.setStorageFormat(STORAGE_FORMAT)).setParameters(parameters);
                table = builder.build();
            } else {
                Table currentTable = getTable();
                checkState(currentMetadataLocation != null, "No current metadata location for existing table");
                String metadataLocation = currentTable.getParameters().get(METADATA_LOCATION);
                if (!currentMetadataLocation.equals(metadataLocation)) {
                    throw new CommitFailedException("Metadata location [%s] is not same as table metadata location [%s] for %s", currentMetadataLocation, metadataLocation, getSchemaTableName());
                }
                table = Table.builder(currentTable).setDataColumns(toHiveColumns(metadata.schema().columns())).withStorage(storage -> storage.setLocation(metadata.location())).setParameter(METADATA_LOCATION, newMetadataLocation).setParameter(PREVIOUS_METADATA_LOCATION, currentMetadataLocation).build();
            }
        } catch (RuntimeException e) {
            try {
                io().deleteFile(newMetadataLocation);
            } catch (RuntimeException exception) {
                e.addSuppressed(exception);
            }
            throw e;
        }
        PrestoPrincipal owner = new PrestoPrincipal(USER, table.getOwner());
        PrincipalPrivileges privileges = new PrincipalPrivileges(ImmutableMultimap.<String, HivePrivilegeInfo>builder().put(table.getOwner(), new HivePrivilegeInfo(SELECT, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(INSERT, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(UPDATE, true, owner, owner)).put(table.getOwner(), new HivePrivilegeInfo(DELETE, true, owner, owner)).build(), ImmutableMultimap.of());
        if (base == null) {
            metastore.createTable(metastoreContext, table, privileges);
        } else {
            metastore.replaceTable(metastoreContext, database, tableName, table, privileges);
        }
    } finally {
        shouldRefresh = true;
        try {
            lockId.ifPresent(id -> metastore.unlock(metastoreContext, id));
        } catch (Exception e) {
            log.error(e, "Failed to unlock: %s", lockId.orElse(null));
        } finally {
            tableLevelMutex.unlock();
        }
    }
}
Also used : ReentrantLock(java.util.concurrent.locks.ReentrantLock) HdfsEnvironment(com.facebook.presto.hive.HdfsEnvironment) LocationProviders(org.apache.iceberg.LocationProviders) LoadingCache(com.google.common.cache.LoadingCache) IcebergUtil.isIcebergTable(com.facebook.presto.iceberg.IcebergUtil.isIcebergTable) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) HiveSchemaUtil(org.apache.iceberg.hive.HiveSchemaUtil) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) ICEBERG_TABLE_TYPE_VALUE(org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE) TABLE_TYPE_PROP(org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP) TableMetadata(org.apache.iceberg.TableMetadata) LocationProvider(org.apache.iceberg.io.LocationProvider) TableOperations(org.apache.iceberg.TableOperations) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) Map(java.util.Map) ICEBERG_INVALID_METADATA(com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA) HdfsContext(com.facebook.presto.hive.HdfsContext) TableMetadataParser(org.apache.iceberg.TableMetadataParser) CommitFailedException(org.apache.iceberg.exceptions.CommitFailedException) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) FileInputFormat(org.apache.hadoop.mapred.FileInputFormat) NestedField(org.apache.iceberg.types.Types.NestedField) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) SELECT(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) CacheLoader(com.google.common.cache.CacheLoader) Objects(java.util.Objects) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) Optional(java.util.Optional) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) CacheBuilder(com.google.common.cache.CacheBuilder) FileOutputFormat(org.apache.hadoop.mapred.FileOutputFormat) TableMetadataParser.getFileExtension(org.apache.iceberg.TableMetadataParser.getFileExtension) Logger(com.facebook.airlift.log.Logger) Table(com.facebook.presto.hive.metastore.Table) Column(com.facebook.presto.hive.metastore.Column) HiveType(com.facebook.presto.hive.HiveType) WRITE_METADATA_LOCATION(org.apache.iceberg.TableProperties.WRITE_METADATA_LOCATION) OutputFile(org.apache.iceberg.io.OutputFile) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) AtomicReference(java.util.concurrent.atomic.AtomicReference) Objects.requireNonNull(java.util.Objects.requireNonNull) TABLE_COMMENT(com.facebook.presto.hive.HiveMetadata.TABLE_COMMENT) METADATA_COMPRESSION_DEFAULT(org.apache.iceberg.TableProperties.METADATA_COMPRESSION_DEFAULT) DELETE(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.DELETE) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) Nullable(javax.annotation.Nullable) USER(com.facebook.presto.spi.security.PrincipalType.USER) INSERT(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT) UPDATE(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE) METADATA_COMPRESSION(org.apache.iceberg.TableProperties.METADATA_COMPRESSION) ReentrantLock(java.util.concurrent.locks.ReentrantLock) LazySimpleSerDe(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe) Integer.parseInt(java.lang.Integer.parseInt) TimeUnit(java.util.concurrent.TimeUnit) UUID.randomUUID(java.util.UUID.randomUUID) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Tasks(org.apache.iceberg.util.Tasks) FileIO(org.apache.iceberg.io.FileIO) NotThreadSafe(javax.annotation.concurrent.NotThreadSafe) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) IcebergUtil.isIcebergTable(com.facebook.presto.iceberg.IcebergUtil.isIcebergTable) Table(com.facebook.presto.hive.metastore.Table) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) HashMap(java.util.HashMap) CommitFailedException(org.apache.iceberg.exceptions.CommitFailedException) PrestoException(com.facebook.presto.spi.PrestoException) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) CommitFailedException(org.apache.iceberg.exceptions.CommitFailedException) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal)

Example 2 with INSERT

use of com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT in project presto by prestodb.

the class ThriftMetastoreUtil method parsePrivilege.

public static Set<HivePrivilegeInfo> parsePrivilege(PrivilegeGrantInfo userGrant, Optional<PrestoPrincipal> grantee) {
    boolean withGrantOption = userGrant.isGrantOption();
    String name = userGrant.getPrivilege().toUpperCase(ENGLISH);
    PrestoPrincipal grantor = new PrestoPrincipal(fromMetastoreApiPrincipalType(userGrant.getGrantorType()), userGrant.getGrantor());
    switch(name) {
        case "ALL":
            return Arrays.stream(HivePrivilegeInfo.HivePrivilege.values()).map(hivePrivilege -> new HivePrivilegeInfo(hivePrivilege, withGrantOption, grantor, grantee.orElse(grantor))).collect(toImmutableSet());
        case "SELECT":
            return ImmutableSet.of(new HivePrivilegeInfo(SELECT, withGrantOption, grantor, grantee.orElse(grantor)));
        case "INSERT":
            return ImmutableSet.of(new HivePrivilegeInfo(INSERT, withGrantOption, grantor, grantee.orElse(grantor)));
        case "UPDATE":
            return ImmutableSet.of(new HivePrivilegeInfo(UPDATE, withGrantOption, grantor, grantee.orElse(grantor)));
        case "DELETE":
            return ImmutableSet.of(new HivePrivilegeInfo(DELETE, withGrantOption, grantor, grantee.orElse(grantor)));
        case "OWNERSHIP":
            return ImmutableSet.of(new HivePrivilegeInfo(OWNERSHIP, withGrantOption, grantor, grantee.orElse(grantor)));
        default:
            throw new IllegalArgumentException("Unsupported privilege name: " + name);
    }
}
Also used : Arrays(java.util.Arrays) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) MetastoreUtil.fromMetastoreDistinctValuesCount(com.facebook.presto.hive.metastore.MetastoreUtil.fromMetastoreDistinctValuesCount) SerDeInfo(org.apache.hadoop.hive.metastore.api.SerDeInfo) EXTERNAL_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE) MATERIALIZED_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.MATERIALIZED_VIEW) HiveColumnStatistics.createDoubleColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) BigDecimal(java.math.BigDecimal) BooleanColumnStatsData(org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData) Math.round(java.lang.Math.round) Map(java.util.Map) DoubleColumnStatsData(org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData) BigInteger(java.math.BigInteger) ENGLISH(java.util.Locale.ENGLISH) EnumSet(java.util.EnumSet) DecimalColumnStatsData(org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData) PartitionMutator(com.facebook.presto.hive.PartitionMutator) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) HIVE_INVALID_METADATA(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA) ColumnStatisticsData.decimalStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.decimalStats) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) Set(java.util.Set) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) SELECT(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT) ColumnConverter(com.facebook.presto.hive.ColumnConverter) ROLE(com.facebook.presto.spi.security.PrincipalType.ROLE) Stream(java.util.stream.Stream) HiveColumnStatistics.createBinaryColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) Date(org.apache.hadoop.hive.metastore.api.Date) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Table(com.facebook.presto.hive.metastore.Table) Database(com.facebook.presto.hive.metastore.Database) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) OptionalLong(java.util.OptionalLong) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) CSV(com.facebook.presto.hive.HiveStorageFormat.CSV) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) DELETE(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.DELETE) LongColumnStatsData(org.apache.hadoop.hive.metastore.api.LongColumnStatsData) PrincipalPrivilegeSet(org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet) PrivilegeGrantInfo(org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo) Nullable(javax.annotation.Nullable) INSERT(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT) ColumnStatisticsData.binaryStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.binaryStats) AbstractIterator(com.google.common.collect.AbstractIterator) PrincipalType(com.facebook.presto.spi.security.PrincipalType) PRESTO_MATERIALIZED_VIEW_FLAG(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_MATERIALIZED_VIEW_FLAG) StringColumnStatsData(org.apache.hadoop.hive.metastore.api.StringColumnStatsData) Strings.emptyToNull(com.google.common.base.Strings.emptyToNull) ColumnStatisticsData.longStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.longStats) DateColumnStatsData(org.apache.hadoop.hive.metastore.api.DateColumnStatsData) TableType(org.apache.hadoop.hive.metastore.TableType) HiveColumnStatistics.createDecimalColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) ArrayDeque(java.util.ArrayDeque) HiveColumnStatistics.createDateColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) ColumnStatisticsData.stringStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.stringStats) ColumnStatisticsData.booleanStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.booleanStats) RolePrincipalGrant(org.apache.hadoop.hive.metastore.api.RolePrincipalGrant) ByteBuffer(java.nio.ByteBuffer) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Locale(java.util.Locale) ColumnStatisticsData.doubleStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.doubleStats) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) Collection(java.util.Collection) Decimal(org.apache.hadoop.hive.metastore.api.Decimal) Order(org.apache.hadoop.hive.metastore.api.Order) Streams(com.google.common.collect.Streams) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) RoleGrant(com.facebook.presto.spi.security.RoleGrant) LocalDate(java.time.LocalDate) Optional(java.util.Optional) HiveColumnStatistics.createBooleanColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) OTHER(com.facebook.presto.hive.metastore.PrestoTableType.OTHER) Queue(java.util.Queue) HiveColumnStatistics.createIntegerColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) Column(com.facebook.presto.hive.metastore.Column) HiveType(com.facebook.presto.hive.HiveType) OptionalDouble(java.util.OptionalDouble) Shorts(com.google.common.primitives.Shorts) PrestoException(com.facebook.presto.spi.PrestoException) Function(java.util.function.Function) Partition(com.facebook.presto.hive.metastore.Partition) OWNERSHIP(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP) HashSet(java.util.HashSet) ColumnStatisticsData.dateStats(org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.dateStats) BinaryColumnStatsData(org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData) Objects.requireNonNull(java.util.Objects.requireNonNull) USER(com.facebook.presto.spi.security.PrincipalType.USER) SelectedRole(com.facebook.presto.spi.security.SelectedRole) Storage(com.facebook.presto.hive.metastore.Storage) HiveColumnStatistics.createStringColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) UPDATE(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE) PRIMITIVE(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE) VIRTUAL_VIEW(com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) FieldSchema(org.apache.hadoop.hive.metastore.api.FieldSchema) Collectors.toList(java.util.stream.Collectors.toList) AVRO_SCHEMA_URL_KEY(com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY) HiveBucketProperty(com.facebook.presto.hive.HiveBucketProperty) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal)

Aggregations

HiveType (com.facebook.presto.hive.HiveType)2 Column (com.facebook.presto.hive.metastore.Column)2 HivePrivilegeInfo (com.facebook.presto.hive.metastore.HivePrivilegeInfo)2 DELETE (com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.DELETE)2 INSERT (com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT)2 SELECT (com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT)2 UPDATE (com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE)2 MetastoreContext (com.facebook.presto.hive.metastore.MetastoreContext)2 PrestoTableType (com.facebook.presto.hive.metastore.PrestoTableType)2 PrincipalPrivileges (com.facebook.presto.hive.metastore.PrincipalPrivileges)2 StorageFormat (com.facebook.presto.hive.metastore.StorageFormat)2 Logger (com.facebook.airlift.log.Logger)1 ColumnConverter (com.facebook.presto.hive.ColumnConverter)1 HdfsContext (com.facebook.presto.hive.HdfsContext)1 HdfsEnvironment (com.facebook.presto.hive.HdfsEnvironment)1 HiveBucketProperty (com.facebook.presto.hive.HiveBucketProperty)1 HIVE_INVALID_METADATA (com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA)1 TABLE_COMMENT (com.facebook.presto.hive.HiveMetadata.TABLE_COMMENT)1 AVRO (com.facebook.presto.hive.HiveStorageFormat.AVRO)1 CSV (com.facebook.presto.hive.HiveStorageFormat.CSV)1