Examples with ConnectorTableProperties io.trino.spi.connector.ConnectorTableProperties used on opensource projects

Example 11 with ConnectorTableProperties

use of io.trino.spi.connector.ConnectorTableProperties in project trino by trinodb.

the class IcebergMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle tableHandle) {
    IcebergTableHandle table = (IcebergTableHandle) tableHandle;
    if (table.getSnapshotId().isEmpty()) {
        // TupleDomain.none() as the predicate
        return new ConnectorTableProperties(TupleDomain.none(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of());
    }
    Table icebergTable = catalog.loadTable(session, table.getSchemaTableName());
    // Extract identity partition fields that are present in all partition specs, for creating the discrete predicates.
    Set<Integer> partitionSourceIds = identityPartitionColumnsInAllSpecs(icebergTable);
    TupleDomain<IcebergColumnHandle> enforcedPredicate = table.getEnforcedPredicate();
    DiscretePredicates discretePredicates = null;
    if (!partitionSourceIds.isEmpty()) {
        // Extract identity partition columns
        Map<Integer, IcebergColumnHandle> columns = getColumns(icebergTable.schema(), typeManager).stream().filter(column -> partitionSourceIds.contains(column.getId())).collect(toImmutableMap(IcebergColumnHandle::getId, Function.identity()));
        Supplier<List<FileScanTask>> lazyFiles = Suppliers.memoize(() -> {
            TableScan tableScan = icebergTable.newScan().useSnapshot(table.getSnapshotId().get()).filter(toIcebergExpression(enforcedPredicate)).includeColumnStats();
            try (CloseableIterable<FileScanTask> iterator = tableScan.planFiles()) {
                return ImmutableList.copyOf(iterator);
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        });
        Iterable<FileScanTask> files = () -> lazyFiles.get().iterator();
        Iterable<TupleDomain<ColumnHandle>> discreteTupleDomain = Iterables.transform(files, fileScan -> {
            // Extract partition values in the data file
            Map<Integer, Optional<String>> partitionColumnValueStrings = getPartitionKeys(fileScan);
            Map<ColumnHandle, NullableValue> partitionValues = partitionSourceIds.stream().filter(partitionColumnValueStrings::containsKey).collect(toImmutableMap(columns::get, columnId -> {
                IcebergColumnHandle column = columns.get(columnId);
                Object prestoValue = deserializePartitionValue(column.getType(), partitionColumnValueStrings.get(columnId).orElse(null), column.getName());
                return NullableValue.of(column.getType(), prestoValue);
            }));
            return TupleDomain.fromFixedValues(partitionValues);
        });
        discretePredicates = new DiscretePredicates(columns.values().stream().map(ColumnHandle.class::cast).collect(toImmutableList()), discreteTupleDomain);
    }
    return new ConnectorTableProperties(// over all tableScan.planFiles() and caching partition values in table handle.
    enforcedPredicate.transformKeys(ColumnHandle.class::cast), // TODO: implement table partitioning
    Optional.empty(), Optional.empty(), Optional.ofNullable(discretePredicates), ImmutableList.of());
}

Example 12 with ConnectorTableProperties

use of io.trino.spi.connector.ConnectorTableProperties in project trino by trinodb.

the class AbstractTestHive method testGetPartitions.

@Test
public void testGetPartitions() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat);
        tableHandle = applyFilter(metadata, tableHandle, Constraint.alwaysTrue());
        ConnectorTableProperties properties = metadata.getTableProperties(newSession(), tableHandle);
        assertExpectedTableProperties(properties, tablePartitionFormatProperties);
        assertExpectedPartitions(tableHandle, tablePartitionFormatPartitions);
    }
}

Example 13 with ConnectorTableProperties

use of io.trino.spi.connector.ConnectorTableProperties in project trino by trinodb.

the class AbstractTestHive method doTestBucketedSortedTableEvolution.

private void doTestBucketedSortedTableEvolution(SchemaTableName tableName) throws Exception {
    int rowCount = 100;
    // Create table and populate it with 3 partitions with different sort orders but same bucketing
    createEmptyTable(tableName, ORC, ImmutableList.of(new Column("id", HIVE_LONG, Optional.empty()), new Column("name", HIVE_STRING, Optional.empty())), ImmutableList.of(new Column("pk", HIVE_STRING, Optional.empty())), Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 4, ImmutableList.of(new SortingColumn("id", ASCENDING), new SortingColumn("name", ASCENDING)))));
    // write a 4-bucket partition sorted by id, name
    MaterializedResult.Builder sortedByIdNameBuilder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> sortedByIdNameBuilder.row((long) i, String.valueOf(i), "sorted_by_id_name"));
    insertData(tableName, sortedByIdNameBuilder.build());
    // write a 4-bucket partition sorted by name
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 4, ImmutableList.of(new SortingColumn("name", ASCENDING)))));
    MaterializedResult.Builder sortedByNameBuilder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> sortedByNameBuilder.row((long) i, String.valueOf(i), "sorted_by_name"));
    insertData(tableName, sortedByNameBuilder.build());
    // write a 4-bucket partition sorted by id
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 4, ImmutableList.of(new SortingColumn("id", ASCENDING)))));
    MaterializedResult.Builder sortedByIdBuilder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> sortedByIdBuilder.row((long) i, String.valueOf(i), "sorted_by_id"));
    insertData(tableName, sortedByIdBuilder.build());
    ConnectorTableHandle tableHandle;
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        metadata.beginQuery(session);
        tableHandle = getTableHandle(metadata, tableName);
        // read entire table
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().collect(toImmutableList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEquals(result.getRowCount(), 300);
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession(ImmutableMap.of("propagate_table_scan_sorting_properties", true));
        metadata.beginQuery(session);
        Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
        // verify local sorting property
        ConnectorTableProperties properties = metadata.getTableProperties(session, tableHandle);
        assertEquals(properties.getLocalProperties(), ImmutableList.of(new SortingProperty<>(columnHandles.get("id"), ASC_NULLS_FIRST)));
        // read on a entire table should fail with exception
        assertThatThrownBy(() -> readTable(transaction, tableHandle, ImmutableList.copyOf(columnHandles.values()), session, TupleDomain.all(), OptionalInt.empty(), Optional.empty())).isInstanceOf(TrinoException.class).hasMessage("Hive table (%s) sorting by [id] is not compatible with partition (pk=sorted_by_name) sorting by [name]." + " This restriction can be avoided by disabling propagate_table_scan_sorting_properties.", tableName);
        // read only the partitions with sorting that is compatible to table sorting
        MaterializedResult result = readTable(transaction, tableHandle, ImmutableList.copyOf(columnHandles.values()), session, TupleDomain.withColumnDomains(ImmutableMap.of(columnHandles.get("pk"), Domain.create(ValueSet.of(VARCHAR, utf8Slice("sorted_by_id_name"), utf8Slice("sorted_by_id")), false))), OptionalInt.empty(), Optional.empty());
        assertEquals(result.getRowCount(), 200);
    }
}

Example 14 with ConnectorTableProperties

use of io.trino.spi.connector.ConnectorTableProperties in project trino by trinodb.

the class MongoMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    MongoTableHandle tableHandle = (MongoTableHandle) table;
    // TODO: sharding key
    Optional<Set<ColumnHandle>> partitioningColumns = Optional.empty();
    ImmutableList.Builder<LocalProperty<ColumnHandle>> localProperties = ImmutableList.builder();
    MongoTable tableInfo = mongoSession.getTable(tableHandle.getSchemaTableName());
    Map<String, ColumnHandle> columns = getColumnHandles(session, tableHandle);
    for (MongoIndex index : tableInfo.getIndexes()) {
        for (MongodbIndexKey key : index.getKeys()) {
            if (key.getSortOrder().isEmpty()) {
                continue;
            }
            if (columns.get(key.getName()) != null) {
                localProperties.add(new SortingProperty<>(columns.get(key.getName()), key.getSortOrder().get()));
            }
        }
    }
    return new ConnectorTableProperties(TupleDomain.all(), Optional.empty(), partitioningColumns, Optional.empty(), localProperties.build());
}

Example 15 with ConnectorTableProperties

use of io.trino.spi.connector.ConnectorTableProperties in project trino by trinodb.

the class PhoenixMetadata method getTableProperties.

@Override
public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) {
    JdbcTableHandle tableHandle = (JdbcTableHandle) table;
    List<LocalProperty<ColumnHandle>> sortingProperties = tableHandle.getSortOrder().map(properties -> properties.stream().map(item -> (LocalProperty<ColumnHandle>) new SortingProperty<ColumnHandle>(item.getColumn(), item.getSortOrder())).collect(toImmutableList())).orElse(ImmutableList.of());
    return new ConnectorTableProperties(TupleDomain.all(), Optional.empty(), Optional.empty(), Optional.empty(), sortingProperties);
}