Search in sources :

Example 21 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class HivePartitionManager method getPartitions.

public HivePartitionResult getPartitions(ConnectorTableHandle tableHandle, List<List<String>> partitionValuesList) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    List<HiveColumnHandle> partitionColumns = hiveTableHandle.getPartitionColumns();
    Optional<HiveBucketHandle> bucketHandle = hiveTableHandle.getBucketHandle();
    List<String> partitionColumnNames = partitionColumns.stream().map(HiveColumnHandle::getName).collect(toImmutableList());
    List<Type> partitionColumnTypes = partitionColumns.stream().map(HiveColumnHandle::getType).collect(toImmutableList());
    List<HivePartition> partitionList = partitionValuesList.stream().map(partitionValues -> toPartitionName(partitionColumnNames, partitionValues)).map(partitionName -> parseValuesAndFilterPartition(tableName, partitionName, partitionColumns, partitionColumnTypes, TupleDomain.all(), value -> true)).map(partition -> partition.orElseThrow(() -> new VerifyException("partition must exist"))).collect(toImmutableList());
    return new HivePartitionResult(partitionColumns, Optional.empty(), partitionList, TupleDomain.all(), TupleDomain.all(), bucketHandle, Optional.empty());
}
Also used : Constraint(io.trino.spi.connector.Constraint) NullableValue(io.trino.spi.predicate.NullableValue) Type(io.trino.spi.type.Type) HiveBucketFilter(io.trino.plugin.hive.util.HiveBucketing.HiveBucketFilter) HiveBucketing.getHiveBucketFilter(io.trino.plugin.hive.util.HiveBucketing.getHiveBucketFilter) Inject(javax.inject.Inject) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) ImmutableList(com.google.common.collect.ImmutableList) HiveUtil.parsePartitionValue(io.trino.plugin.hive.util.HiveUtil.parsePartitionValue) SemiTransactionalHiveMetastore(io.trino.plugin.hive.metastore.SemiTransactionalHiveMetastore) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) Map(java.util.Map) ColumnHandle(io.trino.spi.connector.ColumnHandle) VerifyException(com.google.common.base.VerifyException) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TrinoException(io.trino.spi.TrinoException) TupleDomain(io.trino.spi.predicate.TupleDomain) Sets(com.google.common.collect.Sets) SchemaTableName(io.trino.spi.connector.SchemaTableName) String.format(java.lang.String.format) MetastoreUtil.toPartitionName(io.trino.plugin.hive.metastore.MetastoreUtil.toPartitionName) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) HIVE_EXCEEDED_PARTITION_LIMIT(io.trino.plugin.hive.HiveErrorCode.HIVE_EXCEEDED_PARTITION_LIMIT) MetastoreUtil.computePartitionKeyFilter(io.trino.plugin.hive.metastore.MetastoreUtil.computePartitionKeyFilter) FileUtils(org.apache.hadoop.hive.common.FileUtils) Optional(java.util.Optional) SchemaTableName(io.trino.spi.connector.SchemaTableName) Type(io.trino.spi.type.Type) VerifyException(com.google.common.base.VerifyException)

Example 22 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class SemiTransactionalHiveMetastore method finishUpdate.

public synchronized void finishUpdate(ConnectorSession session, String databaseName, String tableName, Path currentLocation, List<PartitionAndStatementId> partitionAndStatementIds) {
    if (partitionAndStatementIds.isEmpty()) {
        return;
    }
    setShared();
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Action<TableAndMore> oldTableAction = tableActions.get(schemaTableName);
    if (oldTableAction == null) {
        Table table = getExistingTable(schemaTableName.getSchemaName(), schemaTableName.getTableName());
        HdfsContext hdfsContext = new HdfsContext(session);
        PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner().orElseThrow());
        tableActions.put(schemaTableName, new Action<>(ActionType.UPDATE, new TableAndAcidDirectories(table, Optional.of(principalPrivileges), Optional.of(currentLocation), partitionAndStatementIds), hdfsContext, session.getQueryId()));
        return;
    }
    switch(oldTableAction.getType()) {
        case DROP:
            throw new TableNotFoundException(schemaTableName);
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
        case DELETE_ROWS:
        case UPDATE:
            throw new UnsupportedOperationException("Inserting, updating or deleting in a table that was added, altered, inserted into, updated or deleted from in the same transaction is not supported");
        case DROP_PRESERVE_DATA:
            // TODO
            break;
    }
    throw new IllegalStateException("Unknown action type");
}
Also used : TableNotFoundException(io.trino.spi.connector.TableNotFoundException) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 23 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class SemiTransactionalHiveMetastore method getTable.

public synchronized Optional<Table> getTable(String databaseName, String tableName) {
    checkReadable();
    Action<TableAndMore> tableAction = tableActions.get(new SchemaTableName(databaseName, tableName));
    if (tableAction == null) {
        return delegate.getTable(databaseName, tableName);
    }
    switch(tableAction.getType()) {
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
        case DELETE_ROWS:
        case UPDATE:
            return Optional.of(tableAction.getData().getTable());
        case DROP:
            return Optional.empty();
        case DROP_PRESERVE_DATA:
            // TODO
            break;
    }
    throw new IllegalStateException("Unknown action type");
}
Also used : SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 24 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class SemiTransactionalHiveMetastore method dropPartition.

public synchronized void dropPartition(ConnectorSession session, String databaseName, String tableName, List<String> partitionValues, boolean deleteData) {
    setShared();
    Map<List<String>, Action<PartitionAndMore>> partitionActionsOfTable = partitionActions.computeIfAbsent(new SchemaTableName(databaseName, tableName), k -> new HashMap<>());
    Action<PartitionAndMore> oldPartitionAction = partitionActionsOfTable.get(partitionValues);
    if (oldPartitionAction == null) {
        HdfsContext hdfsContext = new HdfsContext(session);
        if (deleteData) {
            partitionActionsOfTable.put(partitionValues, new Action<>(ActionType.DROP, null, hdfsContext, session.getQueryId()));
        } else {
            partitionActionsOfTable.put(partitionValues, new Action<>(ActionType.DROP_PRESERVE_DATA, null, hdfsContext, session.getQueryId()));
        }
        return;
    }
    switch(oldPartitionAction.getType()) {
        case DROP:
        case DROP_PRESERVE_DATA:
            throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues);
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
        case DELETE_ROWS:
        case UPDATE:
            throw new TrinoException(NOT_SUPPORTED, format("dropping a partition added in the same transaction is not supported: %s %s %s", databaseName, tableName, partitionValues));
    }
    throw new IllegalStateException("Unknown action type");
}
Also used : PartitionNotFoundException(io.trino.plugin.hive.PartitionNotFoundException) TrinoException(io.trino.spi.TrinoException) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ArrayList(java.util.ArrayList) ValidTxnWriteIdList(org.apache.hadoop.hive.common.ValidTxnWriteIdList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) LinkedList(java.util.LinkedList) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Example 25 with SchemaTableName

use of io.trino.spi.connector.SchemaTableName in project trino by trinodb.

the class SemiTransactionalHiveMetastore method truncateUnpartitionedTable.

public synchronized void truncateUnpartitionedTable(ConnectorSession session, String databaseName, String tableName) {
    checkReadable();
    Optional<Table> table = getTable(databaseName, tableName);
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    if (table.isEmpty()) {
        throw new TableNotFoundException(schemaTableName);
    }
    if (!table.get().getTableType().equals(MANAGED_TABLE.toString())) {
        throw new TrinoException(NOT_SUPPORTED, "Cannot delete from non-managed Hive table");
    }
    if (!table.get().getPartitionColumns().isEmpty()) {
        throw new IllegalArgumentException("Table is partitioned");
    }
    Path path = new Path(table.get().getStorage().getLocation());
    HdfsContext context = new HdfsContext(session);
    setExclusive((delegate, hdfsEnvironment) -> {
        RecursiveDeleteResult recursiveDeleteResult = recursiveDeleteFiles(hdfsEnvironment, context, path, ImmutableSet.of(""), false);
        if (!recursiveDeleteResult.getNotDeletedEligibleItems().isEmpty()) {
            throw new TrinoException(HIVE_FILESYSTEM_ERROR, format("Error deleting from unpartitioned table %s. These items cannot be deleted: %s", schemaTableName, recursiveDeleteResult.getNotDeletedEligibleItems()));
        }
    });
}
Also used : Path(org.apache.hadoop.fs.Path) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) TrinoException(io.trino.spi.TrinoException) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) SchemaTableName(io.trino.spi.connector.SchemaTableName)

Aggregations

SchemaTableName (io.trino.spi.connector.SchemaTableName)446 Test (org.testng.annotations.Test)212 ImmutableList (com.google.common.collect.ImmutableList)131 ImmutableMap (com.google.common.collect.ImmutableMap)106 List (java.util.List)102 TrinoException (io.trino.spi.TrinoException)100 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)98 ConnectorSession (io.trino.spi.connector.ConnectorSession)98 CatalogSchemaTableName (io.trino.spi.connector.CatalogSchemaTableName)92 TableNotFoundException (io.trino.spi.connector.TableNotFoundException)89 ColumnMetadata (io.trino.spi.connector.ColumnMetadata)86 ConnectorTableMetadata (io.trino.spi.connector.ConnectorTableMetadata)86 Optional (java.util.Optional)78 Map (java.util.Map)67 ColumnHandle (io.trino.spi.connector.ColumnHandle)66 TupleDomain (io.trino.spi.predicate.TupleDomain)59 ConnectorMetadata (io.trino.spi.connector.ConnectorMetadata)58 Path (org.apache.hadoop.fs.Path)55 ConnectorTableHandle (io.trino.spi.connector.ConnectorTableHandle)53 ImmutableSet (com.google.common.collect.ImmutableSet)52