Examples with ConnectorTableHandle io.trino.spi.connector.ConnectorTableHandle used on opensource projects

Example 6 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class MetadataManager method finishRefreshMaterializedView.

@Override
public Optional<ConnectorOutputMetadata> finishRefreshMaterializedView(Session session, TableHandle tableHandle, InsertTableHandle insertHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics, List<TableHandle> sourceTableHandles) {
    CatalogName catalogName = insertHandle.getCatalogName();
    ConnectorMetadata metadata = getMetadata(session, catalogName);
    List<ConnectorTableHandle> sourceConnectorHandles = sourceTableHandles.stream().map(TableHandle::getConnectorHandle).collect(toImmutableList());
    return metadata.finishRefreshMaterializedView(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle(), insertHandle.getConnectorHandle(), fragments, computedStatistics, sourceConnectorHandles);
}

Example 7 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class MetadataManager method beginRefreshMaterializedView.

@Override
public InsertTableHandle beginRefreshMaterializedView(Session session, TableHandle tableHandle, List<TableHandle> sourceTableHandles) {
    CatalogName catalogName = tableHandle.getCatalogName();
    CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName);
    ConnectorMetadata metadata = catalogMetadata.getMetadata(session);
    ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(catalogName);
    List<ConnectorTableHandle> sourceConnectorHandles = sourceTableHandles.stream().map(TableHandle::getConnectorHandle).collect(Collectors.toList());
    sourceConnectorHandles.add(tableHandle.getConnectorHandle());
    if (sourceConnectorHandles.stream().map(Object::getClass).distinct().count() > 1) {
        throw new TrinoException(NOT_SUPPORTED, "Cross connector materialized views are not supported");
    }
    ConnectorInsertTableHandle handle = metadata.beginRefreshMaterializedView(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle(), sourceConnectorHandles, getRetryPolicy(session).getRetryMode());
    return new InsertTableHandle(tableHandle.getCatalogName(), transactionHandle, handle);
}

Example 8 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class HiveMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    HiveTableHandle hiveTable = (HiveTableHandle) table;
    List<ColumnHandle> partitionColumns = ImmutableList.copyOf(hiveTable.getPartitionColumns());
    TupleDomain<ColumnHandle> predicate = TupleDomain.all();
    Optional<DiscretePredicates> discretePredicates = Optional.empty();
    // So computation of  predicate and discretePredicates are not valid.
    if (hiveTable.getPartitionNames().isEmpty()) {
        Optional<List<HivePartition>> partitions = hiveTable.getPartitions().or(() -> {
            // We load the partitions to compute the predicates enforced by the table.
            // Note that the computation is not persisted in the table handle, so can be redone many times
            // TODO: https://github.com/trinodb/trino/issues/10980.
            HivePartitionResult partitionResult = partitionManager.getPartitions(metastore, table, new Constraint(hiveTable.getEnforcedConstraint()));
            if (partitionManager.canPartitionsBeLoaded(partitionResult)) {
                return Optional.of(partitionManager.getPartitionsAsList(partitionResult));
            }
            return Optional.empty();
        });
        if (partitions.isPresent()) {
            List<HivePartition> hivePartitions = partitions.orElseThrow();
            // Since the partitions are fully loaded now, we need to compute
            predicate = createPredicate(partitionColumns, hivePartitions);
            // this check allows us to ensure that table is partitioned
            if (!partitionColumns.isEmpty()) {
                // Do not create tuple domains for every partition at the same time!
                // There can be a huge number of partitions so use an iterable so
                // all domains do not need to be in memory at the same time.
                Iterable<TupleDomain<ColumnHandle>> partitionDomains = Iterables.transform(hivePartitions, hivePartition -> TupleDomain.fromFixedValues(hivePartition.getKeys()));
                discretePredicates = Optional.of(new DiscretePredicates(partitionColumns, partitionDomains));
            }
        }
    }
    Optional<ConnectorTablePartitioning> tablePartitioning = Optional.empty();
    List<LocalProperty<ColumnHandle>> sortingProperties = ImmutableList.of();
    if (hiveTable.getBucketHandle().isPresent()) {
        if (isPropagateTableScanSortingProperties(session) && !hiveTable.getBucketHandle().get().getSortedBy().isEmpty()) {
            // Populating SortingProperty guarantees to the engine that it is reading pre-sorted input.
            // We detect compatibility between table and partition level sorted_by properties
            // and fail the query if there is a mismatch in HiveSplitManager#getPartitionMetadata.
            // This can lead to incorrect results if a sorted_by property is defined over unsorted files.
            Map<String, ColumnHandle> columnHandles = getColumnHandles(session, table);
            sortingProperties = hiveTable.getBucketHandle().get().getSortedBy().stream().map(sortingColumn -> new SortingProperty<>(columnHandles.get(sortingColumn.getColumnName()), sortingColumn.getOrder().getSortOrder())).collect(toImmutableList());
        }
        if (isBucketExecutionEnabled(session)) {
            tablePartitioning = hiveTable.getBucketHandle().map(bucketing -> new ConnectorTablePartitioning(new HivePartitioningHandle(bucketing.getBucketingVersion(), bucketing.getReadBucketCount(), bucketing.getColumns().stream().map(HiveColumnHandle::getHiveType).collect(toImmutableList()), OptionalInt.empty(), false), bucketing.getColumns().stream().map(ColumnHandle.class::cast).collect(toImmutableList())));
        }
    }
    return new ConnectorTableProperties(predicate, tablePartitioning, Optional.empty(), discretePredicates, sortingProperties);
}

Example 9 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class HiveMetadata method beginInsert.

@Override
public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, List<ColumnHandle> columns, RetryMode retryMode) {
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    checkTableIsWritable(table, writesToNonManagedTablesEnabled);
    for (Column column : table.getDataColumns()) {
        if (!isWritableType(column.getType())) {
            throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table %s with column type %s not supported", tableName, column.getType()));
        }
    }
    boolean isTransactional = isTransactionalTable(table.getParameters());
    if (isTransactional && retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Hive transactional tables is not supported with query retries enabled");
    }
    if (isTransactional && !autoCommit) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Hive transactional tables is not supported in explicit transactions (use autocommit mode)");
    }
    if (isSparkBucketedTable(table)) {
        throw new TrinoException(NOT_SUPPORTED, "Inserting into Spark bucketed tables is not supported");
    }
    List<HiveColumnHandle> handles = hiveColumnHandles(table, typeManager, getTimestampPrecision(session)).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableList());
    HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table);
    Optional.ofNullable(table.getParameters().get(SKIP_HEADER_COUNT_KEY)).map(Integer::parseInt).ifPresent(headerSkipCount -> {
        if (headerSkipCount > 1) {
            throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table with value of %s property greater than 1 is not supported", SKIP_HEADER_COUNT_KEY));
        }
    });
    if (table.getParameters().containsKey(SKIP_FOOTER_COUNT_KEY)) {
        throw new TrinoException(NOT_SUPPORTED, format("Inserting into Hive table with %s property not supported", SKIP_FOOTER_COUNT_KEY));
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    AcidTransaction transaction = isTransactional ? metastore.beginInsert(session, table) : NO_ACID_TRANSACTION;
    HiveInsertTableHandle result = new HiveInsertTableHandle(tableName.getSchemaName(), tableName.getTableName(), handles, metastore.generatePageSinkMetadata(tableName), locationHandle, table.getStorage().getBucketProperty(), tableStorageFormat, isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session), transaction, retryMode != NO_RETRIES);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    if (getInsertExistingPartitionsBehavior(session) == InsertExistingPartitionsBehavior.OVERWRITE && writeInfo.getWriteMode() == DIRECT_TO_TARGET_EXISTING_DIRECTORY) {
        if (isTransactional) {
            throw new TrinoException(NOT_SUPPORTED, "Overwriting existing partition in transactional tables doesn't support DIRECT_TO_TARGET_EXISTING_DIRECTORY write mode");
        }
        // Partition overwrite operation is nonatomic thus can't and shouldn't be used in non autocommit context.
        if (!autoCommit) {
            throw new TrinoException(NOT_SUPPORTED, "Overwriting existing partition in non auto commit context doesn't support DIRECT_TO_TARGET_EXISTING_DIRECTORY write mode");
        }
    }
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), tableName);
    return result;
}

Example 10 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class HiveMetadata method getInsertLayout.

@Override
public Optional<ConnectorTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    if (table.getStorage().getBucketProperty().isPresent()) {
        if (!isSupportedBucketing(table)) {
            throw new TrinoException(NOT_SUPPORTED, "Cannot write to a table bucketed on an unsupported type");
        }
    } else // Note: we cannot use hiveTableHandle.isInAcidTransaction() here as transaction is not yet set in HiveTableHandle when getInsertLayout is called
    if (isFullAcidTable(table.getParameters())) {
        table = Table.builder(table).withStorage(storage -> storage.setBucketProperty(Optional.of(new HiveBucketProperty(ImmutableList.of(), HiveBucketing.BucketingVersion.BUCKETING_V2, 1, ImmutableList.of())))).build();
    }
    Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(session, table, typeManager);
    List<Column> partitionColumns = table.getPartitionColumns();
    if (hiveBucketHandle.isEmpty()) {
        // return preferred layout which is partitioned by partition columns
        if (partitionColumns.isEmpty()) {
            return Optional.empty();
        }
        return Optional.of(new ConnectorTableLayout(partitionColumns.stream().map(Column::getName).collect(toImmutableList())));
    }
    HiveBucketProperty bucketProperty = table.getStorage().getBucketProperty().orElseThrow(() -> new NoSuchElementException("Bucket property should be set"));
    if (!bucketProperty.getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) {
        throw new TrinoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled");
    }
    ImmutableList.Builder<String> partitioningColumns = ImmutableList.builder();
    hiveBucketHandle.get().getColumns().stream().map(HiveColumnHandle::getName).forEach(partitioningColumns::add);
    partitionColumns.stream().map(Column::getName).forEach(partitioningColumns::add);
    HivePartitioningHandle partitioningHandle = new HivePartitioningHandle(hiveBucketHandle.get().getBucketingVersion(), hiveBucketHandle.get().getTableBucketCount(), hiveBucketHandle.get().getColumns().stream().map(HiveColumnHandle::getHiveType).collect(toImmutableList()), OptionalInt.of(hiveBucketHandle.get().getTableBucketCount()), !partitionColumns.isEmpty() && isParallelPartitionedBucketedWrites(session));
    return Optional.of(new ConnectorTableLayout(partitioningHandle, partitioningColumns.build()));
}