Search in sources :

Example 26 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class GlueHiveMetastore method dropColumn.

@Override
public void dropColumn(String databaseName, String tableName, String columnName) {
    verifyCanDropColumn(this, databaseName, tableName, columnName);
    Table oldTable = getExistingTable(databaseName, tableName);
    if (oldTable.getColumn(columnName).isEmpty()) {
        SchemaTableName name = new SchemaTableName(databaseName, tableName);
        throw new ColumnNotFoundException(name, columnName);
    }
    ImmutableList.Builder<Column> newDataColumns = ImmutableList.builder();
    oldTable.getDataColumns().stream().filter(fieldSchema -> !fieldSchema.getName().equals(columnName)).forEach(newDataColumns::add);
    Table newTable = Table.builder(oldTable).setDataColumns(newDataColumns.build()).build();
    replaceTable(databaseName, tableName, newTable, null);
}
Also used : ThriftMetastoreUtil.updateStatisticsParameters(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.updateStatisticsParameters) AWSStaticCredentialsProvider(com.amazonaws.auth.AWSStaticCredentialsProvider) UnaryOperator.identity(java.util.function.UnaryOperator.identity) DefaultAWSCredentialsProviderChain(com.amazonaws.auth.DefaultAWSCredentialsProviderChain) USER(io.trino.spi.security.PrincipalType.USER) RequestMetricCollector(com.amazonaws.metrics.RequestMetricCollector) DeleteTableRequest(com.amazonaws.services.glue.model.DeleteTableRequest) ColumnStatisticType(io.trino.spi.statistics.ColumnStatisticType) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) Future(java.util.concurrent.Future) GetDatabasesResult(com.amazonaws.services.glue.model.GetDatabasesResult) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Column(io.trino.plugin.hive.metastore.Column) Map(java.util.Map) PartitionWithStatistics(io.trino.plugin.hive.metastore.PartitionWithStatistics) BatchCreatePartitionRequest(com.amazonaws.services.glue.model.BatchCreatePartitionRequest) GetTablesResult(com.amazonaws.services.glue.model.GetTablesResult) RequestHandler2(com.amazonaws.handlers.RequestHandler2) AcidTransaction(io.trino.plugin.hive.acid.AcidTransaction) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Table(io.trino.plugin.hive.metastore.Table) ConnectorIdentity(io.trino.spi.security.ConnectorIdentity) AmazonServiceException(com.amazonaws.AmazonServiceException) GlueToTrinoConverter.mappedCopy(io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.mappedCopy) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) DeletePartitionRequest(com.amazonaws.services.glue.model.DeletePartitionRequest) Set(java.util.Set) DatabaseInput(com.amazonaws.services.glue.model.DatabaseInput) TableInput(com.amazonaws.services.glue.model.TableInput) UpdateTableRequest(com.amazonaws.services.glue.model.UpdateTableRequest) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) SchemaTableName(io.trino.spi.connector.SchemaTableName) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) BatchUpdatePartitionRequestEntry(com.amazonaws.services.glue.model.BatchUpdatePartitionRequestEntry) PartitionInput(com.amazonaws.services.glue.model.PartitionInput) GlueInputConverter.convertPartition(io.trino.plugin.hive.metastore.glue.converter.GlueInputConverter.convertPartition) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) AWSGlueAsync(com.amazonaws.services.glue.AWSGlueAsync) Partition(io.trino.plugin.hive.metastore.Partition) GetPartitionsRequest(com.amazonaws.services.glue.model.GetPartitionsRequest) Segment(com.amazonaws.services.glue.model.Segment) PartitionStatistics(io.trino.plugin.hive.PartitionStatistics) HivePrincipal(io.trino.plugin.hive.metastore.HivePrincipal) HiveUtil(io.trino.plugin.hive.util.HiveUtil) Iterables(com.google.common.collect.Iterables) GetPartitionResult(com.amazonaws.services.glue.model.GetPartitionResult) Strings.isNullOrEmpty(com.google.common.base.Strings.isNullOrEmpty) PartitionNotFoundException(io.trino.plugin.hive.PartitionNotFoundException) ColumnNotFoundException(io.trino.spi.connector.ColumnNotFoundException) ArrayList(java.util.ArrayList) HiveType(io.trino.plugin.hive.HiveType) OptionalLong(java.util.OptionalLong) Comparators.lexicographical(com.google.common.collect.Comparators.lexicographical) Lists(com.google.common.collect.Lists) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) AlreadyExistsException(com.amazonaws.services.glue.model.AlreadyExistsException) AWSCredentialsProvider(com.amazonaws.auth.AWSCredentialsProvider) CreateTableRequest(com.amazonaws.services.glue.model.CreateTableRequest) SchemaAlreadyExistsException(io.trino.plugin.hive.SchemaAlreadyExistsException) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) CreateDatabaseRequest(com.amazonaws.services.glue.model.CreateDatabaseRequest) HiveWriteUtils(io.trino.plugin.hive.util.HiveWriteUtils) Nullable(javax.annotation.Nullable) Executor(java.util.concurrent.Executor) MoreFutures(io.airlift.concurrent.MoreFutures) PartitionValueList(com.amazonaws.services.glue.model.PartitionValueList) GetTableResult(com.amazonaws.services.glue.model.GetTableResult) RoleGrant(io.trino.spi.security.RoleGrant) ExecutionException(java.util.concurrent.ExecutionException) ClientConfiguration(com.amazonaws.ClientConfiguration) AwsSdkUtil.getPaginatedResults(io.trino.plugin.hive.metastore.glue.AwsSdkUtil.getPaginatedResults) AsyncHandler(com.amazonaws.handlers.AsyncHandler) GetPartitionsResult(com.amazonaws.services.glue.model.GetPartitionsResult) HivePrivilege(io.trino.plugin.hive.metastore.HivePrivilegeInfo.HivePrivilege) ThriftMetastoreUtil.getHiveBasicStatistics(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.getHiveBasicStatistics) MetastoreUtil.makePartitionName(io.trino.plugin.hive.metastore.MetastoreUtil.makePartitionName) HiveUtil.toPartitionValues(io.trino.plugin.hive.util.HiveUtil.toPartitionValues) Database(io.trino.plugin.hive.metastore.Database) GetDatabaseRequest(com.amazonaws.services.glue.model.GetDatabaseRequest) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) CompletionService(java.util.concurrent.CompletionService) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) GetDatabasesRequest(com.amazonaws.services.glue.model.GetDatabasesRequest) Collectors.toMap(java.util.stream.Collectors.toMap) ALREADY_EXISTS(io.trino.spi.StandardErrorCode.ALREADY_EXISTS) Path(org.apache.hadoop.fs.Path) BatchUpdatePartitionResult(com.amazonaws.services.glue.model.BatchUpdatePartitionResult) ImmutableSet(com.google.common.collect.ImmutableSet) AWSGlueAsyncClientBuilder(com.amazonaws.services.glue.AWSGlueAsyncClientBuilder) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) GluePartitionConverter(io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) TrinoException(io.trino.spi.TrinoException) STSAssumeRoleSessionCredentialsProvider(com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider) String.format(java.lang.String.format) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) List(java.util.List) GetTableRequest(com.amazonaws.services.glue.model.GetTableRequest) PartitionError(com.amazonaws.services.glue.model.PartitionError) Entry(java.util.Map.Entry) Optional(java.util.Optional) HivePrivilegeInfo(io.trino.plugin.hive.metastore.HivePrivilegeInfo) UpdateDatabaseRequest(com.amazonaws.services.glue.model.UpdateDatabaseRequest) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) Logger(io.airlift.log.Logger) Type(io.trino.spi.type.Type) Function(java.util.function.Function) Inject(javax.inject.Inject) EndpointConfiguration(com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration) GetPartitionRequest(com.amazonaws.services.glue.model.GetPartitionRequest) Collectors.toCollection(java.util.stream.Collectors.toCollection) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) HIVE_METASTORE_ERROR(io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR) Objects.requireNonNull(java.util.Objects.requireNonNull) GlueInputConverter(io.trino.plugin.hive.metastore.glue.converter.GlueInputConverter) DeleteDatabaseRequest(com.amazonaws.services.glue.model.DeleteDatabaseRequest) Comparator.comparing(java.util.Comparator.comparing) VIRTUAL_VIEW(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW) AwsCurrentRegionHolder.getCurrentRegionFromEC2Metadata(io.trino.plugin.hive.aws.AwsCurrentRegionHolder.getCurrentRegionFromEC2Metadata) BatchGetPartitionRequest(com.amazonaws.services.glue.model.BatchGetPartitionRequest) AmazonWebServiceRequest(com.amazonaws.AmazonWebServiceRequest) BatchCreatePartitionResult(com.amazonaws.services.glue.model.BatchCreatePartitionResult) BasicAWSCredentials(com.amazonaws.auth.BasicAWSCredentials) BatchGetPartitionResult(com.amazonaws.services.glue.model.BatchGetPartitionResult) ErrorDetail(com.amazonaws.services.glue.model.ErrorDetail) TupleDomain(io.trino.spi.predicate.TupleDomain) BatchUpdatePartitionRequest(com.amazonaws.services.glue.model.BatchUpdatePartitionRequest) GlueToTrinoConverter(io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter) GetDatabaseResult(com.amazonaws.services.glue.model.GetDatabaseResult) GetTablesRequest(com.amazonaws.services.glue.model.GetTablesRequest) MetastoreUtil.verifyCanDropColumn(io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn) UpdatePartitionRequest(com.amazonaws.services.glue.model.UpdatePartitionRequest) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) Comparator(java.util.Comparator) Table(io.trino.plugin.hive.metastore.Table) ColumnNotFoundException(io.trino.spi.connector.ColumnNotFoundException) Column(io.trino.plugin.hive.metastore.Column) MetastoreUtil.verifyCanDropColumn(io.trino.plugin.hive.metastore.MetastoreUtil.verifyCanDropColumn) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 27 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class GlueHiveMetastore method propagatePartitionErrorToTrinoException.

private static void propagatePartitionErrorToTrinoException(String databaseName, String tableName, List<PartitionError> partitionErrors) {
    if (partitionErrors != null && !partitionErrors.isEmpty()) {
        ErrorDetail errorDetail = partitionErrors.get(0).getErrorDetail();
        String glueExceptionCode = errorDetail.getErrorCode();
        switch(glueExceptionCode) {
            case "AlreadyExistsException":
                throw new TrinoException(ALREADY_EXISTS, errorDetail.getErrorMessage());
            case "EntityNotFoundException":
                throw new TableNotFoundException(new SchemaTableName(databaseName, tableName), errorDetail.getErrorMessage());
            default:
                throw new TrinoException(HIVE_METASTORE_ERROR, errorDetail.getErrorCode() + ": " + errorDetail.getErrorMessage());
        }
    }
}
Also used : ErrorDetail(com.amazonaws.services.glue.model.ErrorDetail) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) TrinoException(io.trino.spi.TrinoException) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 28 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class GlueHiveMetastore method replaceTable.

@Override
public void replaceTable(String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges) {
    try {
        TableInput newTableInput = GlueInputConverter.convertTable(newTable);
        stats.getUpdateTable().call(() -> glueClient.updateTable(new UpdateTableRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableInput(newTableInput)));
    } catch (EntityNotFoundException e) {
        throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
    } catch (AmazonServiceException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : TableInput(com.amazonaws.services.glue.model.TableInput) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) UpdateTableRequest(com.amazonaws.services.glue.model.UpdateTableRequest) AmazonServiceException(com.amazonaws.AmazonServiceException) TrinoException(io.trino.spi.TrinoException) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 29 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class GlueHiveMetastore method createTable.

@Override
public void createTable(Table table, PrincipalPrivileges principalPrivileges) {
    try {
        TableInput input = GlueInputConverter.convertTable(table);
        stats.getCreateTable().call(() -> glueClient.createTable(new CreateTableRequest().withCatalogId(catalogId).withDatabaseName(table.getDatabaseName()).withTableInput(input)));
    } catch (AlreadyExistsException e) {
        throw new TableAlreadyExistsException(new SchemaTableName(table.getDatabaseName(), table.getTableName()));
    } catch (EntityNotFoundException e) {
        throw new SchemaNotFoundException(table.getDatabaseName());
    } catch (AmazonServiceException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : TableInput(com.amazonaws.services.glue.model.TableInput) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) AlreadyExistsException(com.amazonaws.services.glue.model.AlreadyExistsException) SchemaAlreadyExistsException(io.trino.plugin.hive.SchemaAlreadyExistsException) TableAlreadyExistsException(io.trino.plugin.hive.TableAlreadyExistsException) AmazonServiceException(com.amazonaws.AmazonServiceException) TrinoException(io.trino.spi.TrinoException) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) CreateTableRequest(com.amazonaws.services.glue.model.CreateTableRequest) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 30 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class GlueHiveMetastore method updateTableStatistics.

@Override
public void updateTableStatistics(String databaseName, String tableName, AcidTransaction transaction, Function<PartitionStatistics, PartitionStatistics> update) {
    Table table = getExistingTable(databaseName, tableName);
    if (transaction.isAcidTransactionRunning()) {
        table = Table.builder(table).setWriteId(OptionalLong.of(transaction.getWriteId())).build();
    }
    PartitionStatistics currentStatistics = getTableStatistics(table);
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    try {
        TableInput tableInput = GlueInputConverter.convertTable(table);
        final Map<String, String> statisticsParameters = updateStatisticsParameters(table.getParameters(), updatedStatistics.getBasicStatistics());
        tableInput.setParameters(statisticsParameters);
        table = Table.builder(table).setParameters(statisticsParameters).build();
        stats.getUpdateTable().call(() -> glueClient.updateTable(new UpdateTableRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableInput(tableInput)));
        columnStatisticsProvider.updateTableColumnStatistics(table, updatedStatistics.getColumnStatistics());
    } catch (EntityNotFoundException e) {
        throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
    } catch (AmazonServiceException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : TableInput(com.amazonaws.services.glue.model.TableInput) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Table(io.trino.plugin.hive.metastore.Table) PartitionStatistics(io.trino.plugin.hive.PartitionStatistics) UpdateTableRequest(com.amazonaws.services.glue.model.UpdateTableRequest) AmazonServiceException(com.amazonaws.AmazonServiceException) TrinoException(io.trino.spi.TrinoException) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Aggregations

SchemaTableName (io.trino.spi.connector.SchemaTableName)446 Test (org.testng.annotations.Test)212 ImmutableList (com.google.common.collect.ImmutableList)131 ImmutableMap (com.google.common.collect.ImmutableMap)106 List (java.util.List)102 TrinoException (io.trino.spi.TrinoException)100 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)98 ConnectorSession (io.trino.spi.connector.ConnectorSession)98 CatalogSchemaTableName (io.trino.spi.connector.CatalogSchemaTableName)92 TableNotFoundException (io.trino.spi.connector.TableNotFoundException)89 ColumnMetadata (io.trino.spi.connector.ColumnMetadata)86 ConnectorTableMetadata (io.trino.spi.connector.ConnectorTableMetadata)86 Optional (java.util.Optional)78 Map (java.util.Map)67 ColumnHandle (io.trino.spi.connector.ColumnHandle)66 TupleDomain (io.trino.spi.predicate.TupleDomain)59 ConnectorMetadata (io.trino.spi.connector.ConnectorMetadata)58 Path (org.apache.hadoop.fs.Path)55 ConnectorTableHandle (io.trino.spi.connector.ConnectorTableHandle)53 ImmutableSet (com.google.common.collect.ImmutableSet)52