Examples with TupleDomain io.trino.spi.predicate.TupleDomain used on opensource projects

Example 46 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class IcebergSplitSource method fileMatchesPredicate.

@VisibleForTesting
static boolean fileMatchesPredicate(Map<Integer, Type.PrimitiveType> primitiveTypeForFieldId, TupleDomain<IcebergColumnHandle> dynamicFilterPredicate, @Nullable Map<Integer, ByteBuffer> lowerBounds, @Nullable Map<Integer, ByteBuffer> upperBounds, @Nullable Map<Integer, Long> nullValueCounts) {
    if (dynamicFilterPredicate.isNone()) {
        return false;
    }
    Map<IcebergColumnHandle, Domain> domains = dynamicFilterPredicate.getDomains().orElseThrow();
    for (Map.Entry<IcebergColumnHandle, Domain> domainEntry : domains.entrySet()) {
        IcebergColumnHandle column = domainEntry.getKey();
        Domain domain = domainEntry.getValue();
        int fieldId = column.getId();
        boolean mayContainNulls;
        if (nullValueCounts == null) {
            mayContainNulls = true;
        } else {
            Long nullValueCount = nullValueCounts.get(fieldId);
            mayContainNulls = nullValueCount == null || nullValueCount > 0;
        }
        Type type = primitiveTypeForFieldId.get(fieldId);
        Domain statisticsDomain = domainForStatistics(column.getType(), lowerBounds == null ? null : fromByteBuffer(type, lowerBounds.get(fieldId)), upperBounds == null ? null : fromByteBuffer(type, upperBounds.get(fieldId)), mayContainNulls);
        if (!domain.overlaps(statisticsDomain)) {
            return false;
        }
    }
    return true;
}

Example 47 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestIcebergSplitSource method testIncompleteDynamicFilterTimeout.

@Test(timeOut = 30_000)
public void testIncompleteDynamicFilterTimeout() throws Exception {
    long startMillis = System.currentTimeMillis();
    SchemaTableName schemaTableName = new SchemaTableName("tpch", "nation");
    IcebergTableHandle tableHandle = new IcebergTableHandle(schemaTableName.getSchemaName(), schemaTableName.getTableName(), TableType.DATA, Optional.empty(), TupleDomain.all(), TupleDomain.all(), ImmutableSet.of(), Optional.empty());
    Table nationTable = catalog.loadTable(SESSION, schemaTableName);
    IcebergSplitSource splitSource = new IcebergSplitSource(tableHandle, nationTable.newScan(), Optional.empty(), new DynamicFilter() {

        @Override
        public Set<ColumnHandle> getColumnsCovered() {
            return ImmutableSet.of();
        }

        @Override
        public CompletableFuture<?> isBlocked() {
            return CompletableFuture.runAsync(() -> {
                try {
                    TimeUnit.HOURS.sleep(1);
                } catch (InterruptedException e) {
                    throw new IllegalStateException(e);
                }
            });
        }

        @Override
        public boolean isComplete() {
            return false;
        }

        @Override
        public boolean isAwaitable() {
            return true;
        }

        @Override
        public TupleDomain<ColumnHandle> getCurrentPredicate() {
            return TupleDomain.all();
        }
    }, new Duration(2, SECONDS), alwaysTrue(), new TestingTypeManager(), false);
    ImmutableList.Builder<IcebergSplit> splits = ImmutableList.builder();
    while (!splitSource.isFinished()) {
        splitSource.getNextBatch(null, 100).get().getSplits().stream().map(IcebergSplit.class::cast).forEach(splits::add);
    }
    assertThat(splits.build().size()).isGreaterThan(0);
    assertTrue(splitSource.isFinished());
    assertThat(System.currentTimeMillis() - startMillis).as("IcebergSplitSource failed to wait for dynamicFilteringWaitTimeout").isGreaterThanOrEqualTo(2000);
}

Example 48 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestIcebergProjectionPushdownPlans method testDereferencePushdown.

@Test
public void testDereferencePushdown() {
    String testTable = "test_simple_projection_pushdown" + randomTableSuffix();
    QualifiedObjectName completeTableName = new QualifiedObjectName(CATALOG, SCHEMA, testTable);
    getQueryRunner().execute(format("CREATE TABLE %s (col0, col1) WITH (partitioning = ARRAY['col1']) AS" + " SELECT CAST(row(5, 6) AS row(x bigint, y bigint)) AS col0, 5 AS col1 WHERE false", testTable));
    Session session = getQueryRunner().getDefaultSession();
    Optional<TableHandle> tableHandle = getTableHandle(session, completeTableName);
    assertTrue(tableHandle.isPresent(), "expected the table handle to be present");
    Map<String, ColumnHandle> columns = getColumnHandles(session, completeTableName);
    IcebergColumnHandle column0Handle = (IcebergColumnHandle) columns.get("col0");
    IcebergColumnHandle column1Handle = (IcebergColumnHandle) columns.get("col1");
    IcebergColumnHandle columnX = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(0).getId()), BIGINT, Optional.empty());
    IcebergColumnHandle columnY = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(1).getId()), BIGINT, Optional.empty());
    // Simple Projection pushdown
    assertPlan("SELECT col0.x expr_x, col0.y expr_y FROM " + testTable, any(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(columnX, columnY))), TupleDomain.all(), ImmutableMap.of("col0#x", equalTo(columnX), "col0#y", equalTo(columnY)))));
    // Projection and predicate pushdown
    assertPlan(format("SELECT col0.x FROM %s WHERE col0.x = col1 + 3 and col0.y = 2", testTable), anyTree(filter("y = BIGINT '2' AND (x =  CAST((col1 + 3) AS BIGINT))", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column1Handle, columnX, columnY)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnY, Domain.singleValue(BIGINT, 2L))));
    }, TupleDomain.all(), ImmutableMap.of("y", equalTo(columnY), "x", equalTo(columnX), "col1", equalTo(column1Handle))))));
    // Projection and predicate pushdown with overlapping columns
    assertPlan(format("SELECT col0, col0.y expr_y FROM %s WHERE col0.x = 5", testTable), anyTree(filter("x = BIGINT '5'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column0Handle, columnX)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 5L))));
    }, TupleDomain.all(), ImmutableMap.of("col0", equalTo(column0Handle), "x", equalTo(columnX))))));
    // Projection and predicate pushdown with joins
    assertPlan(format("SELECT T.col0.x, T.col0, T.col0.y FROM %s T join %s S on T.col1 = S.col1 WHERE (T.col0.x = 2)", testTable, testTable), anyTree(project(ImmutableMap.of("expr_0_x", expression("expr_0[1]"), "expr_0", expression("expr_0"), "expr_0_y", expression("expr_0[2]")), join(INNER, ImmutableList.of(equiJoinClause("t_expr_1", "s_expr_1")), anyTree(filter("x = BIGINT '2'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        Set<IcebergColumnHandle> expectedProjections = ImmutableSet.of(column0Handle, column1Handle, columnX);
        TupleDomain<IcebergColumnHandle> expectedUnenforcedConstraint = TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 2L)));
        return icebergTableHandle.getProjectedColumns().equals(expectedProjections) && unenforcedConstraint.equals(expectedUnenforcedConstraint);
    }, TupleDomain.all(), ImmutableMap.of("x", equalTo(columnX), "expr_0", equalTo(column0Handle), "t_expr_1", equalTo(column1Handle))))), anyTree(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(column1Handle))), TupleDomain.all(), ImmutableMap.of("s_expr_1", equalTo(column1Handle))))))));
}

Example 49 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class KuduClientSession method addConstraintPredicates.

/**
 * translates TupleDomain to KuduPredicates.
 */
private void addConstraintPredicates(KuduTable table, KuduScanToken.KuduScanTokenBuilder builder, TupleDomain<ColumnHandle> constraintSummary) {
    verify(!constraintSummary.isNone(), "constraintSummary is none");
    if (constraintSummary.isAll()) {
        return;
    }
    Schema schema = table.getSchema();
    for (TupleDomain.ColumnDomain<ColumnHandle> columnDomain : constraintSummary.getColumnDomains().get()) {
        int position = ((KuduColumnHandle) columnDomain.getColumn()).getOrdinalPosition();
        ColumnSchema columnSchema = schema.getColumnByIndex(position);
        Domain domain = columnDomain.getDomain();
        verify(!domain.isNone(), "Domain is none");
        if (domain.isAll()) {
        // no restriction
        } else if (domain.isOnlyNull()) {
            builder.addPredicate(KuduPredicate.newIsNullPredicate(columnSchema));
        } else if (!domain.getValues().isNone() && domain.isNullAllowed()) {
        // no restriction
        } else if (domain.getValues().isAll() && !domain.isNullAllowed()) {
            builder.addPredicate(KuduPredicate.newIsNotNullPredicate(columnSchema));
        } else if (domain.isSingleValue()) {
            KuduPredicate predicate = createEqualsPredicate(columnSchema, domain.getSingleValue());
            builder.addPredicate(predicate);
        } else {
            ValueSet valueSet = domain.getValues();
            if (valueSet instanceof EquatableValueSet) {
                DiscreteValues discreteValues = valueSet.getDiscreteValues();
                KuduPredicate predicate = createInListPredicate(columnSchema, discreteValues);
                builder.addPredicate(predicate);
            } else if (valueSet instanceof SortedRangeSet) {
                Ranges ranges = ((SortedRangeSet) valueSet).getRanges();
                List<Range> rangeList = ranges.getOrderedRanges();
                if (rangeList.stream().allMatch(Range::isSingleValue)) {
                    io.trino.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
                    List<Object> javaValues = rangeList.stream().map(range -> TypeHelper.getJavaValue(type, range.getSingleValue())).collect(toImmutableList());
                    KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, javaValues);
                    builder.addPredicate(predicate);
                } else {
                    Range span = ranges.getSpan();
                    if (!span.isLowUnbounded()) {
                        KuduPredicate.ComparisonOp op = span.isLowInclusive() ? GREATER_EQUAL : GREATER;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getLowBoundedValue());
                        builder.addPredicate(predicate);
                    }
                    if (!span.isHighUnbounded()) {
                        KuduPredicate.ComparisonOp op = span.isHighInclusive() ? LESS_EQUAL : LESS;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getHighBoundedValue());
                        builder.addPredicate(predicate);
                    }
                }
            } else {
                throw new IllegalStateException("Unexpected domain: " + domain);
            }
        }
    }
}

Example 50 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class RaptorPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, DynamicFilter dynamicFilter) {
    RaptorSplit raptorSplit = (RaptorSplit) split;
    RaptorTableHandle raptorTable = (RaptorTableHandle) table;
    OptionalInt bucketNumber = raptorSplit.getBucketNumber();
    TupleDomain<RaptorColumnHandle> predicate = raptorTable.getConstraint();
    OrcReaderOptions options = new OrcReaderOptions().withMaxMergeDistance(getReaderMaxMergeDistance(session)).withMaxBufferSize(getReaderMaxReadSize(session)).withStreamBufferSize(getReaderStreamBufferSize(session)).withTinyStripeThreshold(getReaderTinyStripeThreshold(session)).withLazyReadSmallRanges(isReaderLazyReadSmallRanges(session));
    OptionalLong transactionId = raptorSplit.getTransactionId();
    if (raptorSplit.getShardUuids().size() == 1) {
        UUID shardUuid = raptorSplit.getShardUuids().iterator().next();
        return createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId);
    }
    Iterator<ConnectorPageSource> iterator = raptorSplit.getShardUuids().stream().map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId)).iterator();
    return new ConcatPageSource(iterator);
}