Examples with Constraint io.trino.spi.connector.Constraint used on opensource projects

Example 26 with Constraint

use of io.trino.spi.connector.Constraint in project trino by trinodb.

the class InformationSchemaMetadata method calculatePrefixesWithSchemaName.

private Set<QualifiedTablePrefix> calculatePrefixesWithSchemaName(ConnectorSession connectorSession, TupleDomain<ColumnHandle> constraint, Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate) {
    Optional<Set<String>> schemas = filterString(constraint, SCHEMA_COLUMN_HANDLE);
    if (schemas.isPresent()) {
        return schemas.get().stream().filter(this::isLowerCase).filter(schema -> predicate.isEmpty() || predicate.get().test(schemaAsFixedValues(schema))).map(schema -> new QualifiedTablePrefix(catalogName, schema)).collect(toImmutableSet());
    }
    if (predicate.isEmpty()) {
        return ImmutableSet.of(new QualifiedTablePrefix(catalogName));
    }
    Session session = ((FullConnectorSession) connectorSession).getSession();
    return listSchemaNames(session).filter(prefix -> predicate.get().test(schemaAsFixedValues(prefix.getSchemaName().get()))).collect(toImmutableSet());
}

Example 27 with Constraint

use of io.trino.spi.connector.Constraint in project trino by trinodb.

the class InformationSchemaMetadata method calculatePrefixesWithTableName.

private Set<QualifiedTablePrefix> calculatePrefixesWithTableName(InformationSchemaTable informationSchemaTable, ConnectorSession connectorSession, Set<QualifiedTablePrefix> prefixes, TupleDomain<ColumnHandle> constraint, Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate) {
    Session session = ((FullConnectorSession) connectorSession).getSession();
    Optional<Set<String>> tables = filterString(constraint, TABLE_NAME_COLUMN_HANDLE);
    if (tables.isPresent()) {
        return prefixes.stream().peek(prefix -> verify(prefix.asQualifiedObjectName().isEmpty())).flatMap(prefix -> prefix.getSchemaName().map(schemaName -> Stream.of(prefix)).orElseGet(() -> listSchemaNames(session))).flatMap(prefix -> tables.get().stream().filter(this::isLowerCase).map(table -> new QualifiedObjectName(catalogName, prefix.getSchemaName().get(), table))).filter(objectName -> {
            if (!isColumnsEnumeratingTable(informationSchemaTable) || metadata.isMaterializedView(session, objectName) || metadata.isView(session, objectName)) {
                return true;
            }
            // This is a columns enumerating table and the object is not a view
            try {
                // filtering in case the source table does not exist or there is a problem with redirection.
                return metadata.getRedirectionAwareTableHandle(session, objectName).getTableHandle().isPresent();
            } catch (TrinoException e) {
                if (e.getErrorCode().equals(TABLE_REDIRECTION_ERROR.toErrorCode())) {
                    // Ignore redirection errors for listing, treat as if the table does not exist
                    return false;
                }
                throw e;
            }
        }).filter(objectName -> predicate.isEmpty() || predicate.get().test(asFixedValues(objectName))).map(QualifiedObjectName::asQualifiedTablePrefix).collect(toImmutableSet());
    }
    if (predicate.isEmpty() || !isColumnsEnumeratingTable(informationSchemaTable)) {
        return prefixes;
    }
    return prefixes.stream().flatMap(prefix -> Stream.concat(metadata.listTables(session, prefix).stream(), metadata.listViews(session, prefix).stream())).filter(objectName -> predicate.get().test(asFixedValues(objectName))).map(QualifiedObjectName::asQualifiedTablePrefix).collect(toImmutableSet());
}

Example 28 with Constraint

use of io.trino.spi.connector.Constraint in project trino by trinodb.

the class DeltaLakeSplitManager method getSplits.

private Stream<DeltaLakeSplit> getSplits(ConnectorTransactionHandle transaction, DeltaLakeTableHandle tableHandle, ConnectorSession session, Optional<DataSize> maxScannedFileSize, Set<ColumnHandle> columnsCoveredByDynamicFilter, Constraint constraint) {
    DeltaLakeMetastore metastore = getMetastore(session, transaction);
    String tableLocation = metastore.getTableLocation(tableHandle.getSchemaTableName(), session);
    List<AddFileEntry> validDataFiles = metastore.getValidDataFiles(tableHandle.getSchemaTableName(), session);
    TupleDomain<DeltaLakeColumnHandle> enforcedPartitionConstraint = tableHandle.getEnforcedPartitionConstraint();
    TupleDomain<DeltaLakeColumnHandle> nonPartitionConstraint = tableHandle.getNonPartitionConstraint();
    // Delta Lake handles updates and deletes by copying entire data files, minus updates/deletes. Because of this we can only have one Split/UpdatablePageSource
    // per file.
    boolean splittable = tableHandle.getWriteType().isEmpty();
    AtomicInteger remainingInitialSplits = new AtomicInteger(maxInitialSplits);
    Optional<Instant> filesModifiedAfter = tableHandle.getAnalyzeHandle().flatMap(AnalyzeHandle::getFilesModifiedAfter);
    Optional<Long> maxScannedFileSizeInBytes = maxScannedFileSize.map(DataSize::toBytes);
    Set<String> predicatedColumnNames = Stream.concat(nonPartitionConstraint.getDomains().orElseThrow().keySet().stream(), columnsCoveredByDynamicFilter.stream().map(DeltaLakeColumnHandle.class::cast)).map(// TODO is DeltaLakeColumnHandle.name normalized?
    column -> column.getName().toLowerCase(ENGLISH)).collect(toImmutableSet());
    List<ColumnMetadata> schema = extractSchema(tableHandle.getMetadataEntry(), typeManager);
    List<ColumnMetadata> predicatedColumns = schema.stream().filter(// ColumnMetadata.name is lowercase
    column -> predicatedColumnNames.contains(column.getName())).collect(toImmutableList());
    return validDataFiles.stream().flatMap(addAction -> {
        if (tableHandle.getAnalyzeHandle().isPresent() && !tableHandle.getAnalyzeHandle().get().isInitialAnalyze() && !addAction.isDataChange()) {
            // skip files which do not introduce data change on non-initial ANALYZE
            return Stream.empty();
        }
        if (filesModifiedAfter.isPresent() && addAction.getModificationTime() <= filesModifiedAfter.get().toEpochMilli()) {
            return Stream.empty();
        }
        if (maxScannedFileSizeInBytes.isPresent() && addAction.getSize() > maxScannedFileSizeInBytes.get()) {
            return Stream.empty();
        }
        Map<DeltaLakeColumnHandle, Domain> enforcedDomains = enforcedPartitionConstraint.getDomains().orElseThrow();
        if (!partitionMatchesPredicate(addAction.getCanonicalPartitionValues(), enforcedDomains)) {
            return Stream.empty();
        }
        TupleDomain<DeltaLakeColumnHandle> statisticsPredicate = createStatisticsPredicate(addAction, predicatedColumns, tableHandle.getMetadataEntry().getCanonicalPartitionColumns());
        if (!nonPartitionConstraint.overlaps(statisticsPredicate)) {
            return Stream.empty();
        }
        if (constraint.predicate().isPresent()) {
            Map<String, Optional<String>> partitionValues = addAction.getCanonicalPartitionValues();
            Map<ColumnHandle, NullableValue> deserializedValues = constraint.getPredicateColumns().orElseThrow().stream().filter(column -> column instanceof DeltaLakeColumnHandle).filter(column -> partitionValues.containsKey(((DeltaLakeColumnHandle) column).getName())).collect(toImmutableMap(identity(), column -> {
                DeltaLakeColumnHandle deltaLakeColumn = (DeltaLakeColumnHandle) column;
                return NullableValue.of(deltaLakeColumn.getType(), deserializePartitionValue(deltaLakeColumn, addAction.getCanonicalPartitionValues().get(deltaLakeColumn.getName())));
            }));
            if (!constraint.predicate().get().test(deserializedValues)) {
                return Stream.empty();
            }
        }
        return splitsForFile(session, addAction, tableLocation, addAction.getCanonicalPartitionValues(), statisticsPredicate, splittable, remainingInitialSplits).stream();
    });
}

Example 29 with Constraint

use of io.trino.spi.connector.Constraint in project trino by trinodb.

the class IcebergMetadata method applyFilter.

@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint) {
    IcebergTableHandle table = (IcebergTableHandle) handle;
    Table icebergTable = catalog.loadTable(session, table.getSchemaTableName());
    Set<Integer> partitionSourceIds = identityPartitionColumnsInAllSpecs(icebergTable);
    BiPredicate<IcebergColumnHandle, Domain> isIdentityPartition = (column, domain) -> partitionSourceIds.contains(column.getId());
    TupleDomain<IcebergColumnHandle> newEnforcedConstraint = constraint.getSummary().transformKeys(IcebergColumnHandle.class::cast).filter(isIdentityPartition).intersect(table.getEnforcedPredicate());
    TupleDomain<IcebergColumnHandle> remainingConstraint = constraint.getSummary().transformKeys(IcebergColumnHandle.class::cast).filter(isIdentityPartition.negate());
    TupleDomain<IcebergColumnHandle> newUnenforcedConstraint = remainingConstraint.filter((columnHandle, predicate) -> !isStructuralType(columnHandle.getType())).intersect(table.getUnenforcedPredicate());
    if (newEnforcedConstraint.equals(table.getEnforcedPredicate()) && newUnenforcedConstraint.equals(table.getUnenforcedPredicate())) {
        return Optional.empty();
    }
    return Optional.of(new ConstraintApplicationResult<>(new IcebergTableHandle(table.getSchemaName(), table.getTableName(), table.getTableType(), table.getSnapshotId(), newUnenforcedConstraint, newEnforcedConstraint, table.getProjectedColumns(), table.getNameMappingJson()), remainingConstraint.transformKeys(ColumnHandle.class::cast), false));
}

Example 30 with Constraint

use of io.trino.spi.connector.Constraint in project trino by trinodb.

the class TestInformationSchemaMetadata method testInformationSchemaPredicatePushdownWithConstraintPredicate.

@Test
public void testInformationSchemaPredicatePushdownWithConstraintPredicate() {
    TransactionId transactionId = transactionManager.beginTransaction(false);
    Constraint constraint = new Constraint(TupleDomain.all(), TestInformationSchemaMetadata::testConstraint, testConstraintColumns());
    ConnectorSession session = createNewSession(transactionId);
    ConnectorMetadata metadata = new InformationSchemaMetadata("test_catalog", this.metadata);
    InformationSchemaTableHandle tableHandle = (InformationSchemaTableHandle) metadata.getTableHandle(session, new SchemaTableName("information_schema", "columns"));
    tableHandle = metadata.applyFilter(session, tableHandle, constraint).map(ConstraintApplicationResult::getHandle).map(InformationSchemaTableHandle.class::cast).orElseThrow(AssertionError::new);
    assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view")));
}