Examples with ConnectorTablePartitioning io.trino.spi.connector.ConnectorTablePartitioning used on opensource projects

Example 1 with ConnectorTablePartitioning

use of io.trino.spi.connector.ConnectorTablePartitioning in project trino by trinodb.

the class HiveMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    HiveTableHandle hiveTable = (HiveTableHandle) table;
    List<ColumnHandle> partitionColumns = ImmutableList.copyOf(hiveTable.getPartitionColumns());
    TupleDomain<ColumnHandle> predicate = TupleDomain.all();
    Optional<DiscretePredicates> discretePredicates = Optional.empty();
    // So computation of  predicate and discretePredicates are not valid.
    if (hiveTable.getPartitionNames().isEmpty()) {
        Optional<List<HivePartition>> partitions = hiveTable.getPartitions().or(() -> {
            // We load the partitions to compute the predicates enforced by the table.
            // Note that the computation is not persisted in the table handle, so can be redone many times
            // TODO: https://github.com/trinodb/trino/issues/10980.
            HivePartitionResult partitionResult = partitionManager.getPartitions(metastore, table, new Constraint(hiveTable.getEnforcedConstraint()));
            if (partitionManager.canPartitionsBeLoaded(partitionResult)) {
                return Optional.of(partitionManager.getPartitionsAsList(partitionResult));
            }
            return Optional.empty();
        });
        if (partitions.isPresent()) {
            List<HivePartition> hivePartitions = partitions.orElseThrow();
            // Since the partitions are fully loaded now, we need to compute
            predicate = createPredicate(partitionColumns, hivePartitions);
            // this check allows us to ensure that table is partitioned
            if (!partitionColumns.isEmpty()) {
                // Do not create tuple domains for every partition at the same time!
                // There can be a huge number of partitions so use an iterable so
                // all domains do not need to be in memory at the same time.
                Iterable<TupleDomain<ColumnHandle>> partitionDomains = Iterables.transform(hivePartitions, hivePartition -> TupleDomain.fromFixedValues(hivePartition.getKeys()));
                discretePredicates = Optional.of(new DiscretePredicates(partitionColumns, partitionDomains));
            }
        }
    }
    Optional<ConnectorTablePartitioning> tablePartitioning = Optional.empty();
    List<LocalProperty<ColumnHandle>> sortingProperties = ImmutableList.of();
    if (hiveTable.getBucketHandle().isPresent()) {
        if (isPropagateTableScanSortingProperties(session) && !hiveTable.getBucketHandle().get().getSortedBy().isEmpty()) {
            // Populating SortingProperty guarantees to the engine that it is reading pre-sorted input.
            // We detect compatibility between table and partition level sorted_by properties
            // and fail the query if there is a mismatch in HiveSplitManager#getPartitionMetadata.
            // This can lead to incorrect results if a sorted_by property is defined over unsorted files.
            Map<String, ColumnHandle> columnHandles = getColumnHandles(session, table);
            sortingProperties = hiveTable.getBucketHandle().get().getSortedBy().stream().map(sortingColumn -> new SortingProperty<>(columnHandles.get(sortingColumn.getColumnName()), sortingColumn.getOrder().getSortOrder())).collect(toImmutableList());
        }
        if (isBucketExecutionEnabled(session)) {
            tablePartitioning = hiveTable.getBucketHandle().map(bucketing -> new ConnectorTablePartitioning(new HivePartitioningHandle(bucketing.getBucketingVersion(), bucketing.getReadBucketCount(), bucketing.getColumns().stream().map(HiveColumnHandle::getHiveType).collect(toImmutableList()), OptionalInt.empty(), false), bucketing.getColumns().stream().map(ColumnHandle.class::cast).collect(toImmutableList())));
        }
    }
    return new ConnectorTableProperties(predicate, tablePartitioning, Optional.empty(), discretePredicates, sortingProperties);
}

Example 2 with ConnectorTablePartitioning

use of io.trino.spi.connector.ConnectorTablePartitioning in project trino by trinodb.

the class TestPushProjectionIntoTableScan method createMockFactory.

private MockConnectorFactory createMockFactory(Map<String, ColumnHandle> assignments, Optional<MockConnectorFactory.ApplyProjection> applyProjection) {
    List<ColumnMetadata> metadata = assignments.entrySet().stream().map(entry -> new ColumnMetadata(entry.getKey(), ((TpchColumnHandle) entry.getValue()).getType())).collect(toImmutableList());
    MockConnectorFactory.Builder builder = MockConnectorFactory.builder().withListSchemaNames(connectorSession -> ImmutableList.of(TEST_SCHEMA)).withListTables((connectorSession, schema) -> TEST_SCHEMA.equals(schema) ? ImmutableList.of(TEST_SCHEMA_TABLE) : ImmutableList.of()).withGetColumns(schemaTableName -> metadata).withGetTableProperties((session, tableHandle) -> {
        MockConnectorTableHandle mockTableHandle = (MockConnectorTableHandle) tableHandle;
        if (mockTableHandle.getTableName().getTableName().equals(TEST_TABLE)) {
            return new ConnectorTableProperties(TupleDomain.all(), Optional.of(new ConnectorTablePartitioning(PARTITIONING_HANDLE, ImmutableList.of(column("col", VARCHAR)))), Optional.empty(), Optional.empty(), ImmutableList.of());
        }
        return new ConnectorTableProperties();
    });
    if (applyProjection.isPresent()) {
        builder = builder.withApplyProjection(applyProjection.get());
    }
    return builder.build();
}

Example 3 with ConnectorTablePartitioning

use of io.trino.spi.connector.ConnectorTablePartitioning in project trino by trinodb.

the class KuduMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    KuduTableHandle handle = (KuduTableHandle) table;
    KuduTable kuduTable = handle.getTable(clientSession);
    Optional<ConnectorTablePartitioning> tablePartitioning = Optional.empty();
    Optional<Set<ColumnHandle>> partitioningColumns = Optional.empty();
    List<LocalProperty<ColumnHandle>> localProperties = ImmutableList.of();
    if (isKuduGroupedExecutionEnabled(session) && isTableSupportGroupedExecution(kuduTable)) {
        Map<String, ColumnHandle> columnMap = getColumnHandles(session, handle);
        List<Integer> bucketColumnIds = getBucketColumnIds(kuduTable);
        List<ColumnHandle> bucketColumns = getSpecifyColumns(kuduTable.getSchema(), bucketColumnIds, columnMap);
        Optional<List<KuduRangePartition>> kuduRangePartitions = getKuduRangePartitions(kuduTable);
        tablePartitioning = Optional.of(new ConnectorTablePartitioning(new KuduPartitioningHandle(handle.getSchemaTableName().getSchemaName(), handle.getSchemaTableName().getTableName(), handle.getBucketCount().orElse(0), bucketColumnIds, bucketColumns.stream().map(KuduColumnHandle.class::cast).map(KuduColumnHandle::getType).collect(Collectors.toList()), kuduRangePartitions), bucketColumns));
        partitioningColumns = Optional.of(ImmutableSet.copyOf(bucketColumns));
    }
    return new ConnectorTableProperties(handle.getConstraint(), tablePartitioning, partitioningColumns, Optional.empty(), localProperties);
}

Example 4 with ConnectorTablePartitioning

use of io.trino.spi.connector.ConnectorTablePartitioning in project trino by trinodb.

the class TpchMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    TpchTableHandle tableHandle = (TpchTableHandle) table;
    Optional<ConnectorTablePartitioning> tablePartitioning = Optional.empty();
    Optional<Set<ColumnHandle>> partitioningColumns = Optional.empty();
    List<LocalProperty<ColumnHandle>> localProperties = ImmutableList.of();
    Map<String, ColumnHandle> columns = getColumnHandles(session, tableHandle);
    if (partitioningEnabled && tableHandle.getTableName().equals(TpchTable.ORDERS.getTableName())) {
        ColumnHandle orderKeyColumn = columns.get(columnNaming.getName(OrderColumn.ORDER_KEY));
        tablePartitioning = Optional.of(new ConnectorTablePartitioning(new TpchPartitioningHandle(TpchTable.ORDERS.getTableName(), calculateTotalRows(OrderGenerator.SCALE_BASE, tableHandle.getScaleFactor())), ImmutableList.of(orderKeyColumn)));
        partitioningColumns = Optional.of(ImmutableSet.of(orderKeyColumn));
        localProperties = ImmutableList.of(new SortingProperty<>(orderKeyColumn, SortOrder.ASC_NULLS_FIRST));
    } else if (partitioningEnabled && tableHandle.getTableName().equals(TpchTable.LINE_ITEM.getTableName())) {
        ColumnHandle orderKeyColumn = columns.get(columnNaming.getName(LineItemColumn.ORDER_KEY));
        tablePartitioning = Optional.of(new ConnectorTablePartitioning(new TpchPartitioningHandle(TpchTable.ORDERS.getTableName(), calculateTotalRows(OrderGenerator.SCALE_BASE, tableHandle.getScaleFactor())), ImmutableList.of(orderKeyColumn)));
        partitioningColumns = Optional.of(ImmutableSet.of(orderKeyColumn));
        localProperties = ImmutableList.of(new SortingProperty<>(orderKeyColumn, SortOrder.ASC_NULLS_FIRST), new SortingProperty<>(columns.get(columnNaming.getName(LineItemColumn.LINE_NUMBER)), SortOrder.ASC_NULLS_FIRST));
    }
    TupleDomain<ColumnHandle> constraint = tableHandle.getConstraint();
    if (predicatePushdownEnabled && constraint.isAll()) {
        if (tableHandle.getTableName().equals(TpchTable.ORDERS.getTableName())) {
            constraint = toTupleDomain(ImmutableMap.of(toColumnHandle(OrderColumn.ORDER_STATUS), orderStatusNullableValues));
        } else if (tableHandle.getTableName().equals(TpchTable.PART.getTableName())) {
            constraint = toTupleDomain(ImmutableMap.of(toColumnHandle(PartColumn.CONTAINER), partContainerNullableValues, toColumnHandle(PartColumn.TYPE), partTypeNullableValues));
        }
    }
    return new ConnectorTableProperties(constraint, tablePartitioning, partitioningColumns, Optional.empty(), localProperties);
}