Search in sources :

Example 46 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class IcebergSplitSource method fileMatchesPredicate.

@VisibleForTesting
static boolean fileMatchesPredicate(Map<Integer, Type.PrimitiveType> primitiveTypeForFieldId, TupleDomain<IcebergColumnHandle> dynamicFilterPredicate, @Nullable Map<Integer, ByteBuffer> lowerBounds, @Nullable Map<Integer, ByteBuffer> upperBounds, @Nullable Map<Integer, Long> nullValueCounts) {
    if (dynamicFilterPredicate.isNone()) {
        return false;
    }
    Map<IcebergColumnHandle, Domain> domains = dynamicFilterPredicate.getDomains().orElseThrow();
    for (Map.Entry<IcebergColumnHandle, Domain> domainEntry : domains.entrySet()) {
        IcebergColumnHandle column = domainEntry.getKey();
        Domain domain = domainEntry.getValue();
        int fieldId = column.getId();
        boolean mayContainNulls;
        if (nullValueCounts == null) {
            mayContainNulls = true;
        } else {
            Long nullValueCount = nullValueCounts.get(fieldId);
            mayContainNulls = nullValueCount == null || nullValueCount > 0;
        }
        Type type = primitiveTypeForFieldId.get(fieldId);
        Domain statisticsDomain = domainForStatistics(column.getType(), lowerBounds == null ? null : fromByteBuffer(type, lowerBounds.get(fieldId)), upperBounds == null ? null : fromByteBuffer(type, upperBounds.get(fieldId)), mayContainNulls);
        if (!domain.overlaps(statisticsDomain)) {
            return false;
        }
    }
    return true;
}
Also used : TypeConverter.toIcebergType(io.trino.plugin.iceberg.TypeConverter.toIcebergType) Type(org.apache.iceberg.types.Type) Domain(io.trino.spi.predicate.Domain) TupleDomain(io.trino.spi.predicate.TupleDomain) Map(java.util.Map) HashMap(java.util.HashMap) Constraint(io.trino.spi.connector.Constraint) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 47 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestIcebergSplitSource method testIncompleteDynamicFilterTimeout.

@Test(timeOut = 30_000)
public void testIncompleteDynamicFilterTimeout() throws Exception {
    long startMillis = System.currentTimeMillis();
    SchemaTableName schemaTableName = new SchemaTableName("tpch", "nation");
    IcebergTableHandle tableHandle = new IcebergTableHandle(schemaTableName.getSchemaName(), schemaTableName.getTableName(), TableType.DATA, Optional.empty(), TupleDomain.all(), TupleDomain.all(), ImmutableSet.of(), Optional.empty());
    Table nationTable = catalog.loadTable(SESSION, schemaTableName);
    IcebergSplitSource splitSource = new IcebergSplitSource(tableHandle, nationTable.newScan(), Optional.empty(), new DynamicFilter() {

        @Override
        public Set<ColumnHandle> getColumnsCovered() {
            return ImmutableSet.of();
        }

        @Override
        public CompletableFuture<?> isBlocked() {
            return CompletableFuture.runAsync(() -> {
                try {
                    TimeUnit.HOURS.sleep(1);
                } catch (InterruptedException e) {
                    throw new IllegalStateException(e);
                }
            });
        }

        @Override
        public boolean isComplete() {
            return false;
        }

        @Override
        public boolean isAwaitable() {
            return true;
        }

        @Override
        public TupleDomain<ColumnHandle> getCurrentPredicate() {
            return TupleDomain.all();
        }
    }, new Duration(2, SECONDS), alwaysTrue(), new TestingTypeManager(), false);
    ImmutableList.Builder<IcebergSplit> splits = ImmutableList.builder();
    while (!splitSource.isFinished()) {
        splitSource.getNextBatch(null, 100).get().getSplits().stream().map(IcebergSplit.class::cast).forEach(splits::add);
    }
    assertThat(splits.build().size()).isGreaterThan(0);
    assertTrue(splitSource.isFinished());
    assertThat(System.currentTimeMillis() - startMillis).as("IcebergSplitSource failed to wait for dynamicFilteringWaitTimeout").isGreaterThanOrEqualTo(2000);
}
Also used : Table(org.apache.iceberg.Table) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) ValueSet(io.trino.spi.predicate.ValueSet) DynamicFilter(io.trino.spi.connector.DynamicFilter) ImmutableList(com.google.common.collect.ImmutableList) Duration(io.airlift.units.Duration) SchemaTableName(io.trino.spi.connector.SchemaTableName) CompletableFuture(java.util.concurrent.CompletableFuture) TupleDomain(io.trino.spi.predicate.TupleDomain) TestingTypeManager(io.trino.spi.type.TestingTypeManager) Test(org.testng.annotations.Test)

Example 48 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestIcebergProjectionPushdownPlans method testDereferencePushdown.

@Test
public void testDereferencePushdown() {
    String testTable = "test_simple_projection_pushdown" + randomTableSuffix();
    QualifiedObjectName completeTableName = new QualifiedObjectName(CATALOG, SCHEMA, testTable);
    getQueryRunner().execute(format("CREATE TABLE %s (col0, col1) WITH (partitioning = ARRAY['col1']) AS" + " SELECT CAST(row(5, 6) AS row(x bigint, y bigint)) AS col0, 5 AS col1 WHERE false", testTable));
    Session session = getQueryRunner().getDefaultSession();
    Optional<TableHandle> tableHandle = getTableHandle(session, completeTableName);
    assertTrue(tableHandle.isPresent(), "expected the table handle to be present");
    Map<String, ColumnHandle> columns = getColumnHandles(session, completeTableName);
    IcebergColumnHandle column0Handle = (IcebergColumnHandle) columns.get("col0");
    IcebergColumnHandle column1Handle = (IcebergColumnHandle) columns.get("col1");
    IcebergColumnHandle columnX = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(0).getId()), BIGINT, Optional.empty());
    IcebergColumnHandle columnY = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(1).getId()), BIGINT, Optional.empty());
    // Simple Projection pushdown
    assertPlan("SELECT col0.x expr_x, col0.y expr_y FROM " + testTable, any(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(columnX, columnY))), TupleDomain.all(), ImmutableMap.of("col0#x", equalTo(columnX), "col0#y", equalTo(columnY)))));
    // Projection and predicate pushdown
    assertPlan(format("SELECT col0.x FROM %s WHERE col0.x = col1 + 3 and col0.y = 2", testTable), anyTree(filter("y = BIGINT '2' AND (x =  CAST((col1 + 3) AS BIGINT))", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column1Handle, columnX, columnY)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnY, Domain.singleValue(BIGINT, 2L))));
    }, TupleDomain.all(), ImmutableMap.of("y", equalTo(columnY), "x", equalTo(columnX), "col1", equalTo(column1Handle))))));
    // Projection and predicate pushdown with overlapping columns
    assertPlan(format("SELECT col0, col0.y expr_y FROM %s WHERE col0.x = 5", testTable), anyTree(filter("x = BIGINT '5'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column0Handle, columnX)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 5L))));
    }, TupleDomain.all(), ImmutableMap.of("col0", equalTo(column0Handle), "x", equalTo(columnX))))));
    // Projection and predicate pushdown with joins
    assertPlan(format("SELECT T.col0.x, T.col0, T.col0.y FROM %s T join %s S on T.col1 = S.col1 WHERE (T.col0.x = 2)", testTable, testTable), anyTree(project(ImmutableMap.of("expr_0_x", expression("expr_0[1]"), "expr_0", expression("expr_0"), "expr_0_y", expression("expr_0[2]")), join(INNER, ImmutableList.of(equiJoinClause("t_expr_1", "s_expr_1")), anyTree(filter("x = BIGINT '2'", tableScan(table -> {
        IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
        TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
        Set<IcebergColumnHandle> expectedProjections = ImmutableSet.of(column0Handle, column1Handle, columnX);
        TupleDomain<IcebergColumnHandle> expectedUnenforcedConstraint = TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 2L)));
        return icebergTableHandle.getProjectedColumns().equals(expectedProjections) && unenforcedConstraint.equals(expectedUnenforcedConstraint);
    }, TupleDomain.all(), ImmutableMap.of("x", equalTo(columnX), "expr_0", equalTo(column0Handle), "t_expr_1", equalTo(column1Handle))))), anyTree(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(column1Handle))), TupleDomain.all(), ImmutableMap.of("s_expr_1", equalTo(column1Handle))))))));
}
Also used : MoreFiles.deleteRecursively(com.google.common.io.MoreFiles.deleteRecursively) PlanMatchPattern.any(io.trino.sql.planner.assertions.PlanMatchPattern.any) INNER(io.trino.sql.planner.plan.JoinNode.Type.INNER) Database(io.trino.plugin.hive.metastore.Database) Test(org.testng.annotations.Test) PlanMatchPattern.filter(io.trino.sql.planner.assertions.PlanMatchPattern.filter) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) ImmutableList(com.google.common.collect.ImmutableList) Files(com.google.common.io.Files) BasePushdownPlanTest(io.trino.sql.planner.assertions.BasePushdownPlanTest) LocalQueryRunner(io.trino.testing.LocalQueryRunner) Map(java.util.Map) ColumnHandle(io.trino.spi.connector.ColumnHandle) FileHiveMetastore.createTestingFileHiveMetastore(io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore) PlanMatchPattern.equiJoinClause(io.trino.sql.planner.assertions.PlanMatchPattern.equiJoinClause) EMPTY_MODULE(com.google.inject.util.Modules.EMPTY_MODULE) PlanMatchPattern.join(io.trino.sql.planner.assertions.PlanMatchPattern.join) TestTable.randomTableSuffix(io.trino.testing.sql.TestTable.randomTableSuffix) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.expression(io.trino.sql.planner.assertions.PlanMatchPattern.expression) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) PrincipalType(io.trino.spi.security.PrincipalType) Domain(io.trino.spi.predicate.Domain) Set(java.util.Set) TupleDomain(io.trino.spi.predicate.TupleDomain) File(java.io.File) String.format(java.lang.String.format) Predicates.equalTo(com.google.common.base.Predicates.equalTo) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) TableHandle(io.trino.metadata.TableHandle) PlanMatchPattern.anyTree(io.trino.sql.planner.assertions.PlanMatchPattern.anyTree) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) BIGINT(io.trino.spi.type.BigintType.BIGINT) PlanMatchPattern.project(io.trino.sql.planner.assertions.PlanMatchPattern.project) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) PlanMatchPattern.tableScan(io.trino.sql.planner.assertions.PlanMatchPattern.tableScan) Session(io.trino.Session) ColumnHandle(io.trino.spi.connector.ColumnHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) TupleDomain(io.trino.spi.predicate.TupleDomain) TableHandle(io.trino.metadata.TableHandle) Session(io.trino.Session) Test(org.testng.annotations.Test) BasePushdownPlanTest(io.trino.sql.planner.assertions.BasePushdownPlanTest)

Example 49 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class KuduClientSession method addConstraintPredicates.

/**
 * translates TupleDomain to KuduPredicates.
 */
private void addConstraintPredicates(KuduTable table, KuduScanToken.KuduScanTokenBuilder builder, TupleDomain<ColumnHandle> constraintSummary) {
    verify(!constraintSummary.isNone(), "constraintSummary is none");
    if (constraintSummary.isAll()) {
        return;
    }
    Schema schema = table.getSchema();
    for (TupleDomain.ColumnDomain<ColumnHandle> columnDomain : constraintSummary.getColumnDomains().get()) {
        int position = ((KuduColumnHandle) columnDomain.getColumn()).getOrdinalPosition();
        ColumnSchema columnSchema = schema.getColumnByIndex(position);
        Domain domain = columnDomain.getDomain();
        verify(!domain.isNone(), "Domain is none");
        if (domain.isAll()) {
        // no restriction
        } else if (domain.isOnlyNull()) {
            builder.addPredicate(KuduPredicate.newIsNullPredicate(columnSchema));
        } else if (!domain.getValues().isNone() && domain.isNullAllowed()) {
        // no restriction
        } else if (domain.getValues().isAll() && !domain.isNullAllowed()) {
            builder.addPredicate(KuduPredicate.newIsNotNullPredicate(columnSchema));
        } else if (domain.isSingleValue()) {
            KuduPredicate predicate = createEqualsPredicate(columnSchema, domain.getSingleValue());
            builder.addPredicate(predicate);
        } else {
            ValueSet valueSet = domain.getValues();
            if (valueSet instanceof EquatableValueSet) {
                DiscreteValues discreteValues = valueSet.getDiscreteValues();
                KuduPredicate predicate = createInListPredicate(columnSchema, discreteValues);
                builder.addPredicate(predicate);
            } else if (valueSet instanceof SortedRangeSet) {
                Ranges ranges = ((SortedRangeSet) valueSet).getRanges();
                List<Range> rangeList = ranges.getOrderedRanges();
                if (rangeList.stream().allMatch(Range::isSingleValue)) {
                    io.trino.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
                    List<Object> javaValues = rangeList.stream().map(range -> TypeHelper.getJavaValue(type, range.getSingleValue())).collect(toImmutableList());
                    KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, javaValues);
                    builder.addPredicate(predicate);
                } else {
                    Range span = ranges.getSpan();
                    if (!span.isLowUnbounded()) {
                        KuduPredicate.ComparisonOp op = span.isLowInclusive() ? GREATER_EQUAL : GREATER;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getLowBoundedValue());
                        builder.addPredicate(predicate);
                    }
                    if (!span.isHighUnbounded()) {
                        KuduPredicate.ComparisonOp op = span.isHighInclusive() ? LESS_EQUAL : LESS;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getHighBoundedValue());
                        builder.addPredicate(predicate);
                    }
                }
            } else {
                throw new IllegalStateException("Unexpected domain: " + domain);
            }
        }
    }
}
Also used : ColumnHandle(io.trino.spi.connector.ColumnHandle) Ranges(io.trino.spi.predicate.Ranges) Schema(org.apache.kudu.Schema) ColumnSchema(org.apache.kudu.ColumnSchema) HashBucketSchema(org.apache.kudu.client.PartitionSchema.HashBucketSchema) EquatableValueSet(io.trino.spi.predicate.EquatableValueSet) ColumnSchema(org.apache.kudu.ColumnSchema) Range(io.trino.spi.predicate.Range) KuduPredicate(org.apache.kudu.client.KuduPredicate) Type(org.apache.kudu.Type) DecimalType(io.trino.spi.type.DecimalType) TupleDomain(io.trino.spi.predicate.TupleDomain) SortedRangeSet(io.trino.spi.predicate.SortedRangeSet) DiscreteValues(io.trino.spi.predicate.DiscreteValues) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) Domain(io.trino.spi.predicate.Domain) TupleDomain(io.trino.spi.predicate.TupleDomain) EquatableValueSet(io.trino.spi.predicate.EquatableValueSet) ValueSet(io.trino.spi.predicate.ValueSet)

Example 50 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class RaptorPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, DynamicFilter dynamicFilter) {
    RaptorSplit raptorSplit = (RaptorSplit) split;
    RaptorTableHandle raptorTable = (RaptorTableHandle) table;
    OptionalInt bucketNumber = raptorSplit.getBucketNumber();
    TupleDomain<RaptorColumnHandle> predicate = raptorTable.getConstraint();
    OrcReaderOptions options = new OrcReaderOptions().withMaxMergeDistance(getReaderMaxMergeDistance(session)).withMaxBufferSize(getReaderMaxReadSize(session)).withStreamBufferSize(getReaderStreamBufferSize(session)).withTinyStripeThreshold(getReaderTinyStripeThreshold(session)).withLazyReadSmallRanges(isReaderLazyReadSmallRanges(session));
    OptionalLong transactionId = raptorSplit.getTransactionId();
    if (raptorSplit.getShardUuids().size() == 1) {
        UUID shardUuid = raptorSplit.getShardUuids().iterator().next();
        return createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId);
    }
    Iterator<ConnectorPageSource> iterator = raptorSplit.getShardUuids().stream().map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId)).iterator();
    return new ConcatPageSource(iterator);
}
Also used : StorageManager(io.trino.plugin.raptor.legacy.storage.StorageManager) RaptorSessionProperties.getReaderTinyStripeThreshold(io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderTinyStripeThreshold) Type(io.trino.spi.type.Type) OptionalInt(java.util.OptionalInt) ConcatPageSource(io.trino.plugin.raptor.legacy.util.ConcatPageSource) Inject(javax.inject.Inject) RaptorSessionProperties.getReaderStreamBufferSize(io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderStreamBufferSize) OptionalLong(java.util.OptionalLong) OrcReaderOptions(io.trino.orc.OrcReaderOptions) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) Objects.requireNonNull(java.util.Objects.requireNonNull) ColumnHandle(io.trino.spi.connector.ColumnHandle) ConnectorPageSource(io.trino.spi.connector.ConnectorPageSource) RaptorSessionProperties.isReaderLazyReadSmallRanges(io.trino.plugin.raptor.legacy.RaptorSessionProperties.isReaderLazyReadSmallRanges) Iterator(java.util.Iterator) RaptorSessionProperties.getReaderMaxMergeDistance(io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxMergeDistance) ConnectorSplit(io.trino.spi.connector.ConnectorSplit) ConnectorPageSourceProvider(io.trino.spi.connector.ConnectorPageSourceProvider) ConnectorSession(io.trino.spi.connector.ConnectorSession) UUID(java.util.UUID) RaptorSessionProperties.getReaderMaxReadSize(io.trino.plugin.raptor.legacy.RaptorSessionProperties.getReaderMaxReadSize) TupleDomain(io.trino.spi.predicate.TupleDomain) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) DynamicFilter(io.trino.spi.connector.DynamicFilter) ConnectorTransactionHandle(io.trino.spi.connector.ConnectorTransactionHandle) ConcatPageSource(io.trino.plugin.raptor.legacy.util.ConcatPageSource) OrcReaderOptions(io.trino.orc.OrcReaderOptions) OptionalLong(java.util.OptionalLong) OptionalInt(java.util.OptionalInt) UUID(java.util.UUID) ConnectorPageSource(io.trino.spi.connector.ConnectorPageSource)

Aggregations

TupleDomain (io.trino.spi.predicate.TupleDomain)97 Domain (io.trino.spi.predicate.Domain)77 Map (java.util.Map)50 ColumnHandle (io.trino.spi.connector.ColumnHandle)48 ImmutableMap (com.google.common.collect.ImmutableMap)43 ImmutableList (com.google.common.collect.ImmutableList)41 List (java.util.List)40 Optional (java.util.Optional)36 Set (java.util.Set)33 Test (org.testng.annotations.Test)33 Objects.requireNonNull (java.util.Objects.requireNonNull)32 ConnectorSession (io.trino.spi.connector.ConnectorSession)29 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)28 ImmutableSet (com.google.common.collect.ImmutableSet)26 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)23 Range (io.trino.spi.predicate.Range)22 String.format (java.lang.String.format)22 ImmutableSet.toImmutableSet (com.google.common.collect.ImmutableSet.toImmutableSet)20 TrinoException (io.trino.spi.TrinoException)20 Type (io.trino.spi.type.Type)19