use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.
the class IcebergSplitSource method fileMatchesPredicate.
@VisibleForTesting
static boolean fileMatchesPredicate(Map<Integer, Type.PrimitiveType> primitiveTypeForFieldId, TupleDomain<IcebergColumnHandle> dynamicFilterPredicate, @Nullable Map<Integer, ByteBuffer> lowerBounds, @Nullable Map<Integer, ByteBuffer> upperBounds, @Nullable Map<Integer, Long> nullValueCounts) {
if (dynamicFilterPredicate.isNone()) {
return false;
}
Map<IcebergColumnHandle, Domain> domains = dynamicFilterPredicate.getDomains().orElseThrow();
for (Map.Entry<IcebergColumnHandle, Domain> domainEntry : domains.entrySet()) {
IcebergColumnHandle column = domainEntry.getKey();
Domain domain = domainEntry.getValue();
int fieldId = column.getId();
boolean mayContainNulls;
if (nullValueCounts == null) {
mayContainNulls = true;
} else {
Long nullValueCount = nullValueCounts.get(fieldId);
mayContainNulls = nullValueCount == null || nullValueCount > 0;
}
Type type = primitiveTypeForFieldId.get(fieldId);
Domain statisticsDomain = domainForStatistics(column.getType(), lowerBounds == null ? null : fromByteBuffer(type, lowerBounds.get(fieldId)), upperBounds == null ? null : fromByteBuffer(type, upperBounds.get(fieldId)), mayContainNulls);
if (!domain.overlaps(statisticsDomain)) {
return false;
}
}
return true;
}
use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.
the class TestIcebergSplitSource method testIncompleteDynamicFilterTimeout.
@Test(timeOut = 30_000)
public void testIncompleteDynamicFilterTimeout() throws Exception {
long startMillis = System.currentTimeMillis();
SchemaTableName schemaTableName = new SchemaTableName("tpch", "nation");
IcebergTableHandle tableHandle = new IcebergTableHandle(schemaTableName.getSchemaName(), schemaTableName.getTableName(), TableType.DATA, Optional.empty(), TupleDomain.all(), TupleDomain.all(), ImmutableSet.of(), Optional.empty());
Table nationTable = catalog.loadTable(SESSION, schemaTableName);
IcebergSplitSource splitSource = new IcebergSplitSource(tableHandle, nationTable.newScan(), Optional.empty(), new DynamicFilter() {
@Override
public Set<ColumnHandle> getColumnsCovered() {
return ImmutableSet.of();
}
@Override
public CompletableFuture<?> isBlocked() {
return CompletableFuture.runAsync(() -> {
try {
TimeUnit.HOURS.sleep(1);
} catch (InterruptedException e) {
throw new IllegalStateException(e);
}
});
}
@Override
public boolean isComplete() {
return false;
}
@Override
public boolean isAwaitable() {
return true;
}
@Override
public TupleDomain<ColumnHandle> getCurrentPredicate() {
return TupleDomain.all();
}
}, new Duration(2, SECONDS), alwaysTrue(), new TestingTypeManager(), false);
ImmutableList.Builder<IcebergSplit> splits = ImmutableList.builder();
while (!splitSource.isFinished()) {
splitSource.getNextBatch(null, 100).get().getSplits().stream().map(IcebergSplit.class::cast).forEach(splits::add);
}
assertThat(splits.build().size()).isGreaterThan(0);
assertTrue(splitSource.isFinished());
assertThat(System.currentTimeMillis() - startMillis).as("IcebergSplitSource failed to wait for dynamicFilteringWaitTimeout").isGreaterThanOrEqualTo(2000);
}
use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.
the class TestIcebergProjectionPushdownPlans method testDereferencePushdown.
@Test
public void testDereferencePushdown() {
String testTable = "test_simple_projection_pushdown" + randomTableSuffix();
QualifiedObjectName completeTableName = new QualifiedObjectName(CATALOG, SCHEMA, testTable);
getQueryRunner().execute(format("CREATE TABLE %s (col0, col1) WITH (partitioning = ARRAY['col1']) AS" + " SELECT CAST(row(5, 6) AS row(x bigint, y bigint)) AS col0, 5 AS col1 WHERE false", testTable));
Session session = getQueryRunner().getDefaultSession();
Optional<TableHandle> tableHandle = getTableHandle(session, completeTableName);
assertTrue(tableHandle.isPresent(), "expected the table handle to be present");
Map<String, ColumnHandle> columns = getColumnHandles(session, completeTableName);
IcebergColumnHandle column0Handle = (IcebergColumnHandle) columns.get("col0");
IcebergColumnHandle column1Handle = (IcebergColumnHandle) columns.get("col1");
IcebergColumnHandle columnX = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(0).getId()), BIGINT, Optional.empty());
IcebergColumnHandle columnY = new IcebergColumnHandle(column0Handle.getColumnIdentity(), column0Handle.getType(), ImmutableList.of(column0Handle.getColumnIdentity().getChildren().get(1).getId()), BIGINT, Optional.empty());
// Simple Projection pushdown
assertPlan("SELECT col0.x expr_x, col0.y expr_y FROM " + testTable, any(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(columnX, columnY))), TupleDomain.all(), ImmutableMap.of("col0#x", equalTo(columnX), "col0#y", equalTo(columnY)))));
// Projection and predicate pushdown
assertPlan(format("SELECT col0.x FROM %s WHERE col0.x = col1 + 3 and col0.y = 2", testTable), anyTree(filter("y = BIGINT '2' AND (x = CAST((col1 + 3) AS BIGINT))", tableScan(table -> {
IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column1Handle, columnX, columnY)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnY, Domain.singleValue(BIGINT, 2L))));
}, TupleDomain.all(), ImmutableMap.of("y", equalTo(columnY), "x", equalTo(columnX), "col1", equalTo(column1Handle))))));
// Projection and predicate pushdown with overlapping columns
assertPlan(format("SELECT col0, col0.y expr_y FROM %s WHERE col0.x = 5", testTable), anyTree(filter("x = BIGINT '5'", tableScan(table -> {
IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
return icebergTableHandle.getProjectedColumns().equals(ImmutableSet.of(column0Handle, columnX)) && unenforcedConstraint.equals(TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 5L))));
}, TupleDomain.all(), ImmutableMap.of("col0", equalTo(column0Handle), "x", equalTo(columnX))))));
// Projection and predicate pushdown with joins
assertPlan(format("SELECT T.col0.x, T.col0, T.col0.y FROM %s T join %s S on T.col1 = S.col1 WHERE (T.col0.x = 2)", testTable, testTable), anyTree(project(ImmutableMap.of("expr_0_x", expression("expr_0[1]"), "expr_0", expression("expr_0"), "expr_0_y", expression("expr_0[2]")), join(INNER, ImmutableList.of(equiJoinClause("t_expr_1", "s_expr_1")), anyTree(filter("x = BIGINT '2'", tableScan(table -> {
IcebergTableHandle icebergTableHandle = (IcebergTableHandle) table;
TupleDomain<IcebergColumnHandle> unenforcedConstraint = icebergTableHandle.getUnenforcedPredicate();
Set<IcebergColumnHandle> expectedProjections = ImmutableSet.of(column0Handle, column1Handle, columnX);
TupleDomain<IcebergColumnHandle> expectedUnenforcedConstraint = TupleDomain.withColumnDomains(ImmutableMap.of(columnX, Domain.singleValue(BIGINT, 2L)));
return icebergTableHandle.getProjectedColumns().equals(expectedProjections) && unenforcedConstraint.equals(expectedUnenforcedConstraint);
}, TupleDomain.all(), ImmutableMap.of("x", equalTo(columnX), "expr_0", equalTo(column0Handle), "t_expr_1", equalTo(column1Handle))))), anyTree(tableScan(equalTo(((IcebergTableHandle) tableHandle.get().getConnectorHandle()).withProjectedColumns(Set.of(column1Handle))), TupleDomain.all(), ImmutableMap.of("s_expr_1", equalTo(column1Handle))))))));
}
use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.
the class KuduClientSession method addConstraintPredicates.
/**
* translates TupleDomain to KuduPredicates.
*/
private void addConstraintPredicates(KuduTable table, KuduScanToken.KuduScanTokenBuilder builder, TupleDomain<ColumnHandle> constraintSummary) {
verify(!constraintSummary.isNone(), "constraintSummary is none");
if (constraintSummary.isAll()) {
return;
}
Schema schema = table.getSchema();
for (TupleDomain.ColumnDomain<ColumnHandle> columnDomain : constraintSummary.getColumnDomains().get()) {
int position = ((KuduColumnHandle) columnDomain.getColumn()).getOrdinalPosition();
ColumnSchema columnSchema = schema.getColumnByIndex(position);
Domain domain = columnDomain.getDomain();
verify(!domain.isNone(), "Domain is none");
if (domain.isAll()) {
// no restriction
} else if (domain.isOnlyNull()) {
builder.addPredicate(KuduPredicate.newIsNullPredicate(columnSchema));
} else if (!domain.getValues().isNone() && domain.isNullAllowed()) {
// no restriction
} else if (domain.getValues().isAll() && !domain.isNullAllowed()) {
builder.addPredicate(KuduPredicate.newIsNotNullPredicate(columnSchema));
} else if (domain.isSingleValue()) {
KuduPredicate predicate = createEqualsPredicate(columnSchema, domain.getSingleValue());
builder.addPredicate(predicate);
} else {
ValueSet valueSet = domain.getValues();
if (valueSet instanceof EquatableValueSet) {
DiscreteValues discreteValues = valueSet.getDiscreteValues();
KuduPredicate predicate = createInListPredicate(columnSchema, discreteValues);
builder.addPredicate(predicate);
} else if (valueSet instanceof SortedRangeSet) {
Ranges ranges = ((SortedRangeSet) valueSet).getRanges();
List<Range> rangeList = ranges.getOrderedRanges();
if (rangeList.stream().allMatch(Range::isSingleValue)) {
io.trino.spi.type.Type type = TypeHelper.fromKuduColumn(columnSchema);
List<Object> javaValues = rangeList.stream().map(range -> TypeHelper.getJavaValue(type, range.getSingleValue())).collect(toImmutableList());
KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, javaValues);
builder.addPredicate(predicate);
} else {
Range span = ranges.getSpan();
if (!span.isLowUnbounded()) {
KuduPredicate.ComparisonOp op = span.isLowInclusive() ? GREATER_EQUAL : GREATER;
KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getLowBoundedValue());
builder.addPredicate(predicate);
}
if (!span.isHighUnbounded()) {
KuduPredicate.ComparisonOp op = span.isHighInclusive() ? LESS_EQUAL : LESS;
KuduPredicate predicate = createComparisonPredicate(columnSchema, op, span.getHighBoundedValue());
builder.addPredicate(predicate);
}
}
} else {
throw new IllegalStateException("Unexpected domain: " + domain);
}
}
}
}
use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.
the class RaptorPageSourceProvider method createPageSource.
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, DynamicFilter dynamicFilter) {
RaptorSplit raptorSplit = (RaptorSplit) split;
RaptorTableHandle raptorTable = (RaptorTableHandle) table;
OptionalInt bucketNumber = raptorSplit.getBucketNumber();
TupleDomain<RaptorColumnHandle> predicate = raptorTable.getConstraint();
OrcReaderOptions options = new OrcReaderOptions().withMaxMergeDistance(getReaderMaxMergeDistance(session)).withMaxBufferSize(getReaderMaxReadSize(session)).withStreamBufferSize(getReaderStreamBufferSize(session)).withTinyStripeThreshold(getReaderTinyStripeThreshold(session)).withLazyReadSmallRanges(isReaderLazyReadSmallRanges(session));
OptionalLong transactionId = raptorSplit.getTransactionId();
if (raptorSplit.getShardUuids().size() == 1) {
UUID shardUuid = raptorSplit.getShardUuids().iterator().next();
return createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId);
}
Iterator<ConnectorPageSource> iterator = raptorSplit.getShardUuids().stream().map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, options, transactionId)).iterator();
return new ConcatPageSource(iterator);
}
Aggregations