Search in sources :

Example 96 with TrinoException

use of io.trino.spi.TrinoException in project trino by trinodb.

the class GlueHiveMetastore method createDatabase.

@Override
public void createDatabase(Database database) {
    if (database.getLocation().isEmpty() && defaultDir.isPresent()) {
        String databaseLocation = new Path(defaultDir.get(), database.getDatabaseName()).toString();
        database = Database.builder(database).setLocation(Optional.of(databaseLocation)).build();
    }
    try {
        DatabaseInput databaseInput = GlueInputConverter.convertDatabase(database);
        stats.getCreateDatabase().call(() -> glueClient.createDatabase(new CreateDatabaseRequest().withCatalogId(catalogId).withDatabaseInput(databaseInput)));
    } catch (AlreadyExistsException e) {
        throw new SchemaAlreadyExistsException(database.getDatabaseName());
    } catch (AmazonServiceException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
    if (database.getLocation().isPresent()) {
        HiveWriteUtils.createDirectory(hdfsContext, hdfsEnvironment, new Path(database.getLocation().get()));
    }
}
Also used : Path(org.apache.hadoop.fs.Path) CreateDatabaseRequest(com.amazonaws.services.glue.model.CreateDatabaseRequest) AlreadyExistsException(com.amazonaws.services.glue.model.AlreadyExistsException) SchemaAlreadyExistsException(io.trino.plugin.hive.SchemaAlreadyExistsException) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) SchemaAlreadyExistsException(io.trino.plugin.hive.SchemaAlreadyExistsException) AmazonServiceException(com.amazonaws.AmazonServiceException) TrinoException(io.trino.spi.TrinoException) DatabaseInput(com.amazonaws.services.glue.model.DatabaseInput)

Example 97 with TrinoException

use of io.trino.spi.TrinoException in project trino by trinodb.

the class GlueHiveMetastore method renameColumn.

@Override
public void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName) {
    Table oldTable = getExistingTable(databaseName, tableName);
    if (oldTable.getPartitionColumns().stream().anyMatch(c -> c.getName().equals(oldColumnName))) {
        throw new TrinoException(NOT_SUPPORTED, "Renaming partition columns is not supported");
    }
    ImmutableList.Builder<Column> newDataColumns = ImmutableList.builder();
    for (Column column : oldTable.getDataColumns()) {
        if (column.getName().equals(oldColumnName)) {
            newDataColumns.add(new Column(newColumnName, column.getType(), column.getComment()));
        } else {
            newDataColumns.add(column);
        }
    }
    Table newTable = Table.builder(oldTable).setDataColumns(newDataColumns.build()).build();
    replaceTable(databaseName, tableName, newTable, null);
}
Also used : Table(io.trino.plugin.hive.metastore.Table) Column(io.trino.plugin.hive.metastore.Column) MetastoreUtil.verifyCanDropColumn(io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) TrinoException(io.trino.spi.TrinoException)

Example 98 with TrinoException

use of io.trino.spi.TrinoException in project trino by trinodb.

the class ProtoUtils method fromProto.

public static Table fromProto(alluxio.grpc.table.TableInfo table) {
    if (!table.hasLayout()) {
        throw new TrinoException(NOT_SUPPORTED, "Unsupported table metadata. missing layout.: " + table.getTableName());
    }
    Layout layout = table.getLayout();
    if (!alluxio.table.ProtoUtils.isHiveLayout(layout)) {
        throw new TrinoException(NOT_SUPPORTED, "Unsupported table layout: " + layout + " for table: " + table.getTableName());
    }
    try {
        PartitionInfo partitionInfo = alluxio.table.ProtoUtils.toHiveLayout(layout);
        // compute the data columns
        Set<String> partitionColumns = table.getPartitionColsList().stream().map(FieldSchema::getName).collect(toImmutableSet());
        List<FieldSchema> dataColumns = table.getSchema().getColsList().stream().filter((f) -> !partitionColumns.contains(f.getName())).collect(toImmutableList());
        Map<String, String> tableParameters = table.getParametersMap();
        Table.Builder builder = Table.builder().setDatabaseName(table.getDbName()).setTableName(table.getTableName()).setOwner(Optional.ofNullable(table.getOwner())).setTableType(table.getType().toString()).setDataColumns(dataColumns.stream().map(ProtoUtils::fromProto).collect(toImmutableList())).setPartitionColumns(table.getPartitionColsList().stream().map(ProtoUtils::fromProto).collect(toImmutableList())).setParameters(tableParameters).setViewOriginalText(Optional.empty()).setViewExpandedText(Optional.empty());
        alluxio.grpc.table.layout.hive.Storage storage = partitionInfo.getStorage();
        builder.getStorageBuilder().setSkewed(storage.getSkewed()).setStorageFormat(fromProto(storage.getStorageFormat())).setLocation(storage.getLocation()).setBucketProperty(storage.hasBucketProperty() ? fromProto(tableParameters, storage.getBucketProperty()) : Optional.empty()).setSerdeParameters(storage.getStorageFormat().getSerdelibParametersMap());
        return builder.build();
    } catch (InvalidProtocolBufferException e) {
        throw new IllegalArgumentException("Failed to extract PartitionInfo from TableInfo", e);
    }
}
Also used : Date(alluxio.grpc.table.Date) HiveColumnStatistics.createStringColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) HiveColumnStatistics.createBinaryColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) Database(io.trino.plugin.hive.metastore.Database) HiveColumnStatistics.createDecimalColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) DoubleColumnStatsData(alluxio.grpc.table.DoubleColumnStatsData) BigDecimal(java.math.BigDecimal) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) HiveColumnStatistics.createDateColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) Column(io.trino.plugin.hive.metastore.Column) HiveBucketing(io.trino.plugin.hive.util.HiveBucketing) Map(java.util.Map) BigInteger(java.math.BigInteger) ThriftMetastoreUtil.fromMetastoreDistinctValuesCount(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreDistinctValuesCount) StorageFormat(io.trino.plugin.hive.metastore.StorageFormat) Table(io.trino.plugin.hive.metastore.Table) BinaryColumnStatsData(alluxio.grpc.table.BinaryColumnStatsData) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Layout(alluxio.grpc.table.Layout) Set(java.util.Set) TrinoException(io.trino.spi.TrinoException) DecimalColumnStatsData(alluxio.grpc.table.DecimalColumnStatsData) HiveColumnStatistics.createDoubleColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) PartitionInfo(alluxio.grpc.table.layout.hive.PartitionInfo) List(java.util.List) HiveColumnStatistics.createBooleanColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) LocalDate(java.time.LocalDate) Optional(java.util.Optional) Partition(io.trino.plugin.hive.metastore.Partition) PrincipalType(alluxio.grpc.table.PrincipalType) DateColumnStatsData(alluxio.grpc.table.DateColumnStatsData) OptionalDouble(java.util.OptionalDouble) LongColumnStatsData(alluxio.grpc.table.LongColumnStatsData) HiveBucketProperty(io.trino.plugin.hive.HiveBucketProperty) HiveType(io.trino.plugin.hive.HiveType) OptionalLong(java.util.OptionalLong) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) HIVE_INVALID_METADATA(io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA) FieldSchema(alluxio.grpc.table.FieldSchema) Lists(com.google.common.collect.Lists) ThriftMetastoreUtil.getTotalSizeInBytes(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.getTotalSizeInBytes) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Nullable(javax.annotation.Nullable) ColumnStatisticsData(alluxio.grpc.table.ColumnStatisticsData) InvalidProtocolBufferException(alluxio.shaded.client.com.google.protobuf.InvalidProtocolBufferException) HiveColumnStatistics.createIntegerColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) BooleanColumnStatsData(alluxio.grpc.table.BooleanColumnStatsData) Decimal(alluxio.grpc.table.Decimal) SortingColumn(io.trino.plugin.hive.metastore.SortingColumn) StringColumnStatsData(alluxio.grpc.table.StringColumnStatsData) ThriftMetastoreUtil.fromMetastoreNullsCount(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreNullsCount) Table(io.trino.plugin.hive.metastore.Table) FieldSchema(alluxio.grpc.table.FieldSchema) InvalidProtocolBufferException(alluxio.shaded.client.com.google.protobuf.InvalidProtocolBufferException) Layout(alluxio.grpc.table.Layout) TrinoException(io.trino.spi.TrinoException) PartitionInfo(alluxio.grpc.table.layout.hive.PartitionInfo)

Example 99 with TrinoException

use of io.trino.spi.TrinoException in project trino by trinodb.

the class FileHiveMetastore method addPartitions.

@Override
public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(partitions, "partitions is null");
    Table table = getRequiredTable(databaseName, tableName);
    TableType tableType = TableType.valueOf(table.getTableType());
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType);
    try {
        Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
        for (PartitionWithStatistics partitionWithStatistics : partitions) {
            Partition partition = partitionWithStatistics.getPartition();
            verifiedPartition(table, partition);
            Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
            Path schemaPath = getSchemaPath(PARTITION, partitionMetadataDirectory);
            if (metadataFileSystem.exists(schemaPath)) {
                throw new TrinoException(HIVE_METASTORE_ERROR, "Partition already exists");
            }
            byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partitionWithStatistics));
            schemaFiles.put(schemaPath, schemaJson);
        }
        Set<Path> createdFiles = new LinkedHashSet<>();
        try {
            for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) {
                try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) {
                    createdFiles.add(entry.getKey());
                    outputStream.write(entry.getValue());
                } catch (IOException e) {
                    throw new TrinoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e);
                }
            }
        } catch (Throwable e) {
            for (Path createdFile : createdFiles) {
                try {
                    metadataFileSystem.delete(createdFile, false);
                } catch (IOException ignored) {
                }
            }
            throw e;
        }
    } catch (IOException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) LinkedHashSet(java.util.LinkedHashSet) Partition(io.trino.plugin.hive.metastore.Partition) Table(io.trino.plugin.hive.metastore.Table) HiveUtil.isIcebergTable(io.trino.plugin.hive.util.HiveUtil.isIcebergTable) TableType(org.apache.hadoop.hive.metastore.TableType) OutputStream(java.io.OutputStream) IOException(java.io.IOException) LinkedHashMap(java.util.LinkedHashMap) PartitionWithStatistics(io.trino.plugin.hive.metastore.PartitionWithStatistics) TrinoException(io.trino.spi.TrinoException)

Example 100 with TrinoException

use of io.trino.spi.TrinoException in project trino by trinodb.

the class FileHiveMetastore method setTablePrivileges.

private synchronized void setTablePrivileges(HivePrincipal grantee, String databaseName, String tableName, Collection<HivePrivilegeInfo> privileges) {
    requireNonNull(grantee, "grantee is null");
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(privileges, "privileges is null");
    try {
        Table table = getRequiredTable(databaseName, tableName);
        Path permissionsDirectory = getPermissionsDirectory(table);
        boolean created = metadataFileSystem.mkdirs(permissionsDirectory);
        if (!created && !metadataFileSystem.isDirectory(permissionsDirectory)) {
            throw new TrinoException(HIVE_METASTORE_ERROR, "Could not create permissions directory");
        }
        Path permissionFilePath = getPermissionsPath(permissionsDirectory, grantee);
        List<PermissionMetadata> permissions = privileges.stream().map(hivePrivilegeInfo -> new PermissionMetadata(hivePrivilegeInfo.getHivePrivilege(), hivePrivilegeInfo.isGrantOption(), grantee)).collect(toList());
        writeFile("permissions", permissionFilePath, permissionsCodec, permissions, true);
    } catch (IOException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) ThriftMetastoreUtil.updateStatisticsParameters(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.updateStatisticsParameters) Arrays(java.util.Arrays) FileSystem(org.apache.hadoop.fs.FileSystem) USER(io.trino.spi.security.PrincipalType.USER) FileStatus(org.apache.hadoop.fs.FileStatus) ColumnStatisticType(io.trino.spi.statistics.ColumnStatisticType) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) DATABASE(io.trino.plugin.hive.metastore.file.FileHiveMetastore.SchemaType.DATABASE) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Column(io.trino.plugin.hive.metastore.Column) Map(java.util.Map) PartitionWithStatistics(io.trino.plugin.hive.metastore.PartitionWithStatistics) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) EnumSet(java.util.EnumSet) TABLE_COMMENT(io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT) AcidTransaction(io.trino.plugin.hive.acid.AcidTransaction) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Table(io.trino.plugin.hive.metastore.Table) ConnectorIdentity(io.trino.spi.security.ConnectorIdentity) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) ThreadSafe(javax.annotation.concurrent.ThreadSafe) GuardedBy(javax.annotation.concurrent.GuardedBy) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) SchemaTableName(io.trino.spi.connector.SchemaTableName) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) OWNERSHIP(io.trino.plugin.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP) HdfsConfig(io.trino.plugin.hive.HdfsConfig) ByteStreams(com.google.common.io.ByteStreams) HdfsConfigurationInitializer(io.trino.plugin.hive.HdfsConfigurationInitializer) Partition(io.trino.plugin.hive.metastore.Partition) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) PartitionStatistics(io.trino.plugin.hive.PartitionStatistics) HivePrincipal(io.trino.plugin.hive.metastore.HivePrincipal) PartitionNotFoundException(io.trino.plugin.hive.PartitionNotFoundException) ColumnNotFoundException(io.trino.spi.connector.ColumnNotFoundException) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) HiveType(io.trino.plugin.hive.HiveType) ThriftMetastoreUtil(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) NodeVersion(io.trino.plugin.hive.NodeVersion) SchemaAlreadyExistsException(io.trino.plugin.hive.SchemaAlreadyExistsException) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) LinkedHashSet(java.util.LinkedHashSet) HiveUtil.isIcebergTable(io.trino.plugin.hive.util.HiveUtil.isIcebergTable) SPARK_TABLE_PROVIDER_KEY(io.trino.plugin.hive.util.HiveUtil.SPARK_TABLE_PROVIDER_KEY) IOException(java.io.IOException) HdfsConfiguration(io.trino.plugin.hive.HdfsConfiguration) RoleGrant(io.trino.spi.security.RoleGrant) File(java.io.File) TableType(org.apache.hadoop.hive.metastore.TableType) ArrayDeque(java.util.ArrayDeque) HivePrivilege(io.trino.plugin.hive.metastore.HivePrivilegeInfo.HivePrivilege) HivePartitionManager.extractPartitionValues(io.trino.plugin.hive.HivePartitionManager.extractPartitionValues) ThriftMetastoreUtil.getHiveBasicStatistics(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.getHiveBasicStatistics) MetastoreUtil.makePartitionName(io.trino.plugin.hive.metastore.MetastoreUtil.makePartitionName) HiveUtil.toPartitionValues(io.trino.plugin.hive.util.HiveUtil.toPartitionValues) EXTERNAL_TABLE(org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE) Database(io.trino.plugin.hive.metastore.Database) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) NoHdfsAuthentication(io.trino.plugin.hive.authentication.NoHdfsAuthentication) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) MATERIALIZED_VIEW(org.apache.hadoop.hive.metastore.TableType.MATERIALIZED_VIEW) Locale(java.util.Locale) ALREADY_EXISTS(io.trino.spi.StandardErrorCode.ALREADY_EXISTS) Path(org.apache.hadoop.fs.Path) HiveHdfsConfiguration(io.trino.plugin.hive.HiveHdfsConfiguration) Collectors.toSet(java.util.stream.Collectors.toSet) HiveBasicStatistics(io.trino.plugin.hive.HiveBasicStatistics) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) TrinoException(io.trino.spi.TrinoException) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) Objects(java.util.Objects) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) List(java.util.List) PARTITION(io.trino.plugin.hive.metastore.file.FileHiveMetastore.SchemaType.PARTITION) Entry(java.util.Map.Entry) Optional(java.util.Optional) HivePrivilegeInfo(io.trino.plugin.hive.metastore.HivePrivilegeInfo) FileUtils.unescapePathName(org.apache.hadoop.hive.common.FileUtils.unescapePathName) JsonCodec(io.airlift.json.JsonCodec) VERSION_COMPATIBILITY_CONFIG(io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VERSION_COMPATIBILITY_CONFIG) MetastoreConfig(io.trino.plugin.hive.metastore.MetastoreConfig) Type(io.trino.spi.type.Type) HashMap(java.util.HashMap) Function(java.util.function.Function) HashSet(java.util.HashSet) Builder(com.google.common.collect.ImmutableSet.Builder) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) ImmutableList(com.google.common.collect.ImmutableList) HIVE_METASTORE_ERROR(io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR) UNSAFE_ASSUME_COMPATIBILITY(io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VersionCompatibility.UNSAFE_ASSUME_COMPATIBILITY) Objects.requireNonNull(java.util.Objects.requireNonNull) VIRTUAL_VIEW(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW) VersionCompatibility(io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VersionCompatibility) OutputStream(java.io.OutputStream) DELTA_LAKE_PROVIDER(io.trino.plugin.hive.util.HiveUtil.DELTA_LAKE_PROVIDER) TupleDomain(io.trino.spi.predicate.TupleDomain) ROLE(io.trino.spi.security.PrincipalType.ROLE) Collectors.toList(java.util.stream.Collectors.toList) MetastoreUtil.verifyCanDropColumn(io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn) TABLE(io.trino.plugin.hive.metastore.file.FileHiveMetastore.SchemaType.TABLE) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) Table(io.trino.plugin.hive.metastore.Table) HiveUtil.isIcebergTable(io.trino.plugin.hive.util.HiveUtil.isIcebergTable) TrinoException(io.trino.spi.TrinoException) IOException(java.io.IOException)

Aggregations

TrinoException (io.trino.spi.TrinoException)623 IOException (java.io.IOException)151 ImmutableList (com.google.common.collect.ImmutableList)105 List (java.util.List)100 Type (io.trino.spi.type.Type)93 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)90 SchemaTableName (io.trino.spi.connector.SchemaTableName)83 Path (org.apache.hadoop.fs.Path)83 Optional (java.util.Optional)79 ArrayList (java.util.ArrayList)77 Map (java.util.Map)76 ImmutableMap (com.google.common.collect.ImmutableMap)70 Objects.requireNonNull (java.util.Objects.requireNonNull)69 ConnectorSession (io.trino.spi.connector.ConnectorSession)63 TableNotFoundException (io.trino.spi.connector.TableNotFoundException)62 ImmutableSet (com.google.common.collect.ImmutableSet)56 VarcharType (io.trino.spi.type.VarcharType)54 Set (java.util.Set)54 Slice (io.airlift.slice.Slice)53 Table (io.trino.plugin.hive.metastore.Table)52