use of io.trino.spi.predicate.Domain in project trino by trinodb.
the class ShardPredicate method create.
public static ShardPredicate create(TupleDomain<RaptorColumnHandle> tupleDomain) {
StringJoiner predicate = new StringJoiner(" AND ").setEmptyValue("true");
ImmutableList.Builder<JDBCType> types = ImmutableList.builder();
ImmutableList.Builder<Object> values = ImmutableList.builder();
for (Entry<RaptorColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
Domain domain = entry.getValue();
if (domain.isNullAllowed() || domain.isAll()) {
continue;
}
RaptorColumnHandle handle = entry.getKey();
Type type = handle.getColumnType();
JDBCType jdbcType = jdbcType(type);
if (jdbcType == null) {
continue;
}
if (handle.isShardUuid()) {
predicate.add(createShardPredicate(types, values, domain, jdbcType));
continue;
}
if (!domain.getType().isOrderable()) {
continue;
}
StringJoiner columnPredicate = new StringJoiner(" OR ", "(", ")").setEmptyValue("true");
Ranges ranges = domain.getValues().getRanges();
// prevent generating complicated metadata queries
if (ranges.getRangeCount() > MAX_RANGE_COUNT) {
continue;
}
for (Range range : ranges.getOrderedRanges()) {
String min;
String max;
if (handle.isBucketNumber()) {
min = "bucket_number";
max = "bucket_number";
} else {
min = minColumn(handle.getColumnId());
max = maxColumn(handle.getColumnId());
}
StringJoiner rangePredicate = new StringJoiner(" AND ", "(", ")").setEmptyValue("true");
if (!range.isLowUnbounded()) {
rangePredicate.add(format("(%s >= ? OR %s IS NULL)", max, max));
types.add(jdbcType);
values.add(range.getLowBoundedValue());
}
if (!range.isHighUnbounded()) {
rangePredicate.add(format("(%s <= ? OR %s IS NULL)", min, min));
types.add(jdbcType);
values.add(range.getHighBoundedValue());
}
columnPredicate.add(rangePredicate.toString());
}
predicate.add(columnPredicate.toString());
}
return new ShardPredicate(predicate.toString(), types.build(), values.build());
}
use of io.trino.spi.predicate.Domain in project trino by trinodb.
the class TestDynamicFiltersCollector method testDynamicFiltersCollector.
@Test
public void testDynamicFiltersCollector() {
DynamicFilterId filter = new DynamicFilterId("filter");
DynamicFiltersCollector collector = new DynamicFiltersCollector(() -> {
});
VersionedDynamicFilterDomains domains = collector.acknowledgeAndGetNewDomains(INITIAL_DYNAMIC_FILTERS_VERSION);
// there should be no domains initially
assertEquals(domains.getVersion(), INITIAL_DYNAMIC_FILTERS_VERSION);
assertEquals(domains.getDynamicFilterDomains(), ImmutableMap.of());
Domain initialDomain = multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L));
collector.updateDomains(ImmutableMap.of(filter, initialDomain));
domains = collector.acknowledgeAndGetNewDomains(INITIAL_DYNAMIC_FILTERS_VERSION);
assertEquals(domains.getVersion(), 1L);
assertEquals(domains.getDynamicFilterDomains(), ImmutableMap.of(filter, initialDomain));
// make sure domains are still available when requested again with old version
domains = collector.acknowledgeAndGetNewDomains(INITIAL_DYNAMIC_FILTERS_VERSION);
assertEquals(domains.getVersion(), 1L);
assertEquals(domains.getDynamicFilterDomains(), ImmutableMap.of(filter, initialDomain));
// make sure domains are intersected
collector.updateDomains(ImmutableMap.of(filter, multipleValues(BIGINT, ImmutableList.of(2L))));
collector.updateDomains(ImmutableMap.of(filter, multipleValues(BIGINT, ImmutableList.of(3L, 4L))));
domains = collector.acknowledgeAndGetNewDomains(1L);
assertEquals(domains.getVersion(), 3L);
assertEquals(domains.getDynamicFilterDomains(), ImmutableMap.of(filter, Domain.none(BIGINT)));
// make sure old domains are removed
DynamicFilterId filter2 = new DynamicFilterId("filter2");
collector.updateDomains(ImmutableMap.of(filter2, singleValue(BIGINT, 1L)));
domains = collector.acknowledgeAndGetNewDomains(3L);
assertEquals(domains.getVersion(), 4L);
assertEquals(domains.getDynamicFilterDomains(), ImmutableMap.of(filter2, singleValue(BIGINT, 1L)));
}
use of io.trino.spi.predicate.Domain in project trino by trinodb.
the class AbstractTestHive method testGetPartitionSplitsTableOfflinePartition.
@Test
public void testGetPartitionSplitsTableOfflinePartition() {
try (Transaction transaction = newTransaction()) {
ConnectorMetadata metadata = transaction.getMetadata();
ConnectorSession session = newSession();
metadata.beginQuery(session);
ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition);
assertNotNull(tableHandle);
ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds");
assertNotNull(dsColumn);
Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30"));
TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain));
tableHandle = applyFilter(metadata, tableHandle, new Constraint(tupleDomain));
try {
getSplitCount(getSplits(splitManager, transaction, session, tableHandle));
fail("Expected PartitionOfflineException");
} catch (PartitionOfflineException e) {
assertEquals(e.getTableName(), tableOfflinePartition);
assertEquals(e.getPartition(), "ds=2012-12-30");
}
}
}
use of io.trino.spi.predicate.Domain in project trino by trinodb.
the class AbstractTestHive method doTestTransactionDeleteInsert.
private void doTestTransactionDeleteInsert(HiveStorageFormat storageFormat, SchemaTableName tableName, Domain domainToDrop, MaterializedResult insertData, MaterializedResult expectedData, TransactionDeleteInsertTestTag tag, boolean expectQuerySucceed, Optional<ConflictTrigger> conflictTrigger) throws Exception {
Path writePath = null;
Path targetPath = null;
try (Transaction transaction = newTransaction()) {
try {
ConnectorMetadata metadata = transaction.getMetadata();
ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
ConnectorSession session;
rollbackIfEquals(tag, ROLLBACK_RIGHT_AWAY);
// Query 1: delete
session = newSession();
HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("pk2");
TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumnHandle, domainToDrop));
Constraint constraint = new Constraint(tupleDomain, tupleDomain.asPredicate(), tupleDomain.getDomains().orElseThrow().keySet());
tableHandle = applyFilter(metadata, tableHandle, constraint);
tableHandle = metadata.applyDelete(session, tableHandle).get();
metadata.executeDelete(session, tableHandle);
rollbackIfEquals(tag, ROLLBACK_AFTER_DELETE);
// Query 2: insert
session = newSession();
ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle, ImmutableList.of(), NO_RETRIES);
rollbackIfEquals(tag, ROLLBACK_AFTER_BEGIN_INSERT);
writePath = getStagingPathRoot(insertTableHandle);
targetPath = getTargetPathRoot(insertTableHandle);
ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
sink.appendPage(insertData.toPage());
rollbackIfEquals(tag, ROLLBACK_AFTER_APPEND_PAGE);
Collection<Slice> fragments = getFutureValue(sink.finish());
rollbackIfEquals(tag, ROLLBACK_AFTER_SINK_FINISH);
metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of());
rollbackIfEquals(tag, ROLLBACK_AFTER_FINISH_INSERT);
assertEquals(tag, COMMIT);
if (conflictTrigger.isPresent()) {
JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toList());
conflictTrigger.get().triggerConflict(session, tableName, insertTableHandle, partitionUpdates);
}
transaction.commit();
if (conflictTrigger.isPresent()) {
assertTrue(expectQuerySucceed);
conflictTrigger.get().verifyAndCleanup(session, tableName);
}
} catch (TestingRollbackException e) {
transaction.rollback();
} catch (TrinoException e) {
assertFalse(expectQuerySucceed);
if (conflictTrigger.isPresent()) {
conflictTrigger.get().verifyAndCleanup(newSession(), tableName);
}
}
}
// check that temporary files are removed
if (writePath != null && !writePath.equals(targetPath)) {
HdfsContext context = new HdfsContext(newSession());
FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, writePath);
assertFalse(fileSystem.exists(writePath));
}
try (Transaction transaction = newTransaction()) {
// verify partitions
List<String> partitionNames = transaction.getMetastore().getPartitionNames(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new AssertionError("Table does not exist: " + tableName));
assertEqualsIgnoreOrder(partitionNames, expectedData.getMaterializedRows().stream().map(row -> format("pk1=%s/pk2=%s", row.getField(1), row.getField(2))).distinct().collect(toImmutableList()));
// load the new table
ConnectorSession session = newSession();
ConnectorMetadata metadata = transaction.getMetadata();
metadata.beginQuery(session);
ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
// verify the data
MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedData.getMaterializedRows());
}
}
use of io.trino.spi.predicate.Domain in project trino by trinodb.
the class TestHttpRemoteTask method testDynamicFilters.
@Test(timeOut = 30000)
public void testDynamicFilters() throws Exception {
DynamicFilterId filterId1 = new DynamicFilterId("df1");
DynamicFilterId filterId2 = new DynamicFilterId("df2");
SymbolAllocator symbolAllocator = new SymbolAllocator();
Symbol symbol1 = symbolAllocator.newSymbol("DF_SYMBOL1", BIGINT);
Symbol symbol2 = symbolAllocator.newSymbol("DF_SYMBOL2", BIGINT);
SymbolReference df1 = symbol1.toSymbolReference();
SymbolReference df2 = symbol2.toSymbolReference();
ColumnHandle handle1 = new TestingColumnHandle("column1");
ColumnHandle handle2 = new TestingColumnHandle("column2");
QueryId queryId = new QueryId("test");
TestingTaskResource testingTaskResource = new TestingTaskResource(new AtomicLong(System.nanoTime()), FailureScenario.NO_FAILURE);
DynamicFilterService dynamicFilterService = new DynamicFilterService(PLANNER_CONTEXT.getMetadata(), PLANNER_CONTEXT.getFunctionManager(), new TypeOperators(), newDirectExecutorService());
HttpRemoteTaskFactory httpRemoteTaskFactory = createHttpRemoteTaskFactory(testingTaskResource, dynamicFilterService);
RemoteTask remoteTask = createRemoteTask(httpRemoteTaskFactory, ImmutableSet.of());
Map<DynamicFilterId, Domain> initialDomain = ImmutableMap.of(filterId1, Domain.singleValue(BIGINT, 1L));
testingTaskResource.setInitialTaskInfo(remoteTask.getTaskInfo());
testingTaskResource.setDynamicFilterDomains(new VersionedDynamicFilterDomains(1L, initialDomain));
dynamicFilterService.registerQuery(queryId, TEST_SESSION, ImmutableSet.of(filterId1, filterId2), ImmutableSet.of(filterId1, filterId2), ImmutableSet.of());
dynamicFilterService.stageCannotScheduleMoreTasks(new StageId(queryId, 1), 0, 1);
DynamicFilter dynamicFilter = dynamicFilterService.createDynamicFilter(queryId, ImmutableList.of(new DynamicFilters.Descriptor(filterId1, df1), new DynamicFilters.Descriptor(filterId2, df2)), ImmutableMap.of(symbol1, handle1, symbol2, handle2), symbolAllocator.getTypes());
// make sure initial dynamic filters are collected
CompletableFuture<?> future = dynamicFilter.isBlocked();
remoteTask.start();
future.get();
assertEquals(dynamicFilter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(handle1, Domain.singleValue(BIGINT, 1L))));
assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 1);
// make sure dynamic filters are not collected for every status update
assertEventually(new Duration(15, SECONDS), () -> assertGreaterThanOrEqual(testingTaskResource.getStatusFetchCounter(), 3L));
assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 1L, testingTaskResource.getDynamicFiltersFetchRequests().toString());
future = dynamicFilter.isBlocked();
testingTaskResource.setDynamicFilterDomains(new VersionedDynamicFilterDomains(2L, ImmutableMap.of(filterId2, Domain.singleValue(BIGINT, 2L))));
future.get();
assertEquals(dynamicFilter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(handle1, Domain.singleValue(BIGINT, 1L), handle2, Domain.singleValue(BIGINT, 2L))));
assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 2L, testingTaskResource.getDynamicFiltersFetchRequests().toString());
assertGreaterThanOrEqual(testingTaskResource.getStatusFetchCounter(), 4L);
httpRemoteTaskFactory.stop();
dynamicFilterService.stop();
}
Aggregations