Examples with DynamicFilterId io.trino.sql.planner.plan.DynamicFilterId used on opensource projects

Example 11 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFilterConsumer method testMultiplePartitionsAndColumns.

@Test
public void testMultiplePartitionsAndColumns() throws Exception {
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(ImmutableMap.of(new DynamicFilterId("123"), 0, new DynamicFilterId("456"), 1), ImmutableMap.of(new DynamicFilterId("123"), INTEGER, new DynamicFilterId("456"), BIGINT), 2);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0, new DynamicFilterId("456"), 1));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<DynamicFilterId, Domain>> result = filter.getDynamicFilterDomains();
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 10L), new DynamicFilterId("456"), Domain.singleValue(BIGINT, 100L))));
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 20L), new DynamicFilterId("456"), Domain.singleValue(BIGINT, 200L))));
    assertEquals(result.get(), ImmutableMap.of(new DynamicFilterId("123"), Domain.multipleValues(INTEGER, ImmutableList.of(10L, 20L)), new DynamicFilterId("456"), Domain.multipleValues(BIGINT, ImmutableList.of(100L, 200L))));
}

Example 12 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFilterConsumer method testMultiplePartitions.

@Test
public void testMultiplePartitions() throws Exception {
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(ImmutableMap.of(new DynamicFilterId("123"), 0), ImmutableMap.of(new DynamicFilterId("123"), INTEGER), 2);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<DynamicFilterId, Domain>> result = filter.getDynamicFilterDomains();
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 10L))));
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 20L))));
    assertEquals(result.get(), ImmutableMap.of(new DynamicFilterId("123"), Domain.multipleValues(INTEGER, ImmutableList.of(10L, 20L))));
}

Example 13 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class LocalDynamicFiltersCollector method createDynamicFilter.

// Called during TableScan planning (no need to be synchronized as local planning is single threaded)
public DynamicFilter createDynamicFilter(List<Descriptor> descriptors, Map<Symbol, ColumnHandle> columnsMap, TypeProvider typeProvider, PlannerContext plannerContext) {
    Multimap<DynamicFilterId, Descriptor> descriptorMap = extractSourceSymbols(descriptors);
    // Iterate over dynamic filters that are collected (correspond to one of the futures), and required for filtering (correspond to one of the descriptors).
    // It is possible that some dynamic filters are collected in a different stage - and will not available here.
    // It is also possible that not all local dynamic filters are needed for this specific table scan.
    List<ListenableFuture<TupleDomain<ColumnHandle>>> predicateFutures = descriptorMap.keySet().stream().filter(futures.keySet()::contains).map(filterId -> {
        // Probe-side columns that can be filtered with this dynamic filter resulting domain.
        return Futures.transform(requireNonNull(futures.get(filterId), () -> format("Missing dynamic filter %s", filterId)), // Construct a probe-side predicate by duplicating the resulting domain over the corresponding columns.
        domain -> TupleDomain.withColumnDomains(descriptorMap.get(filterId).stream().collect(toImmutableMap(descriptor -> {
            Symbol probeSymbol = Symbol.from(descriptor.getInput());
            return requireNonNull(columnsMap.get(probeSymbol), () -> format("Missing probe column for %s", probeSymbol));
        }, descriptor -> {
            Type targetType = typeProvider.get(Symbol.from(descriptor.getInput()));
            Domain updatedDomain = descriptor.applyComparison(domain);
            if (!updatedDomain.getType().equals(targetType)) {
                return applySaturatedCasts(plannerContext.getMetadata(), plannerContext.getFunctionManager(), plannerContext.getTypeOperators(), session, updatedDomain, targetType);
            }
            return updatedDomain;
        }))), directExecutor());
    }).collect(toImmutableList());
    Set<ColumnHandle> columnsCovered = descriptorMap.values().stream().map(Descriptor::getInput).map(Symbol::from).map(probeSymbol -> requireNonNull(columnsMap.get(probeSymbol), () -> "Missing probe column for " + probeSymbol)).collect(toImmutableSet());
    return new TableSpecificDynamicFilter(columnsCovered, predicateFutures);
}

Example 14 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(StageInfo outputStageInfo, QueryStats queryStats, ValuePrinter valuePrinter, boolean verbose) {
    Map<PlanNodeId, TableInfo> tableInfos = getAllStages(Optional.of(outputStageInfo)).stream().map(StageInfo::getTables).map(Map::entrySet).flatMap(Collection::stream).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = getAllStages(Optional.of(outputStageInfo));
    List<PlanFragment> allFragments = allStages.stream().map(StageInfo::getPlan).collect(toImmutableList());
    Map<PlanNodeId, PlanNodeStats> aggregatedStats = aggregateStageStats(allStages);
    Map<DynamicFilterId, DynamicFilterDomainStats> dynamicFilterDomainStats = queryStats.getDynamicFiltersStats().getDynamicFilterDomainStats().stream().collect(toImmutableMap(DynamicFilterDomainStats::getDynamicFilterId, identity()));
    TypeProvider typeProvider = getTypeProvider(allFragments);
    for (StageInfo stageInfo : allStages) {
        builder.append(formatFragment(tableScanNode -> tableInfos.get(tableScanNode.getId()), dynamicFilterDomainStats, valuePrinter, stageInfo.getPlan(), Optional.of(stageInfo), Optional.of(aggregatedStats), verbose, typeProvider));
    }
    return builder.toString();
}

Example 15 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestDynamicFilterService method testDynamicFilterConsumerCallbackCount.

@Test
public void testDynamicFilterConsumerCallbackCount() {
    DynamicFilterService dynamicFilterService = createDynamicFilterService();
    DynamicFilterId filterId1 = new DynamicFilterId("df1");
    DynamicFilterId filterId2 = new DynamicFilterId("df2");
    Set<DynamicFilterId> dynamicFilters = ImmutableSet.of(filterId1, filterId2);
    QueryId queryId = new QueryId("query");
    StageId stageId = new StageId(queryId, 0);
    dynamicFilterService.registerQuery(queryId, session, dynamicFilters, dynamicFilters, ImmutableSet.of());
    dynamicFilterService.stageCannotScheduleMoreTasks(stageId, 0, 2);
    Map<DynamicFilterId, Domain> consumerCollectedFilters = new HashMap<>();
    AtomicInteger callbackCount = new AtomicInteger();
    dynamicFilterService.registerDynamicFilterConsumer(queryId, 0, dynamicFilters, domains -> {
        callbackCount.getAndIncrement();
        domains.forEach((filter, domain) -> assertNull(consumerCollectedFilters.put(filter, domain)));
    });
    assertTrue(consumerCollectedFilters.isEmpty());
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId, 0, 0), ImmutableMap.of(filterId1, singleValue(INTEGER, 1L), filterId2, singleValue(INTEGER, 2L)));
    assertTrue(consumerCollectedFilters.isEmpty());
    // complete both filterId1 and filterId2
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId, 1, 0), ImmutableMap.of(filterId1, singleValue(INTEGER, 3L), filterId2, singleValue(INTEGER, 4L)));
    assertEquals(consumerCollectedFilters, ImmutableMap.of(filterId1, multipleValues(INTEGER, ImmutableList.of(1L, 3L)), filterId2, multipleValues(INTEGER, ImmutableList.of(2L, 4L))));
    // both filters should be received in single callback
    assertEquals(callbackCount.get(), 1);
    // register another consumer after both filters have been collected
    Map<DynamicFilterId, Domain> secondConsumerCollectedFilters = new HashMap<>();
    AtomicInteger secondCallbackCount = new AtomicInteger();
    dynamicFilterService.registerDynamicFilterConsumer(queryId, 0, dynamicFilters, domains -> {
        secondCallbackCount.getAndIncrement();
        domains.forEach((filter, domain) -> assertNull(secondConsumerCollectedFilters.put(filter, domain)));
    });
    assertEquals(secondConsumerCollectedFilters, ImmutableMap.of(filterId1, multipleValues(INTEGER, ImmutableList.of(1L, 3L)), filterId2, multipleValues(INTEGER, ImmutableList.of(2L, 4L))));
    // both filters should be received by second consumer in single callback
    assertEquals(secondCallbackCount.get(), 1);
    // first consumer should not receive callback again since it already got the completed filter
    assertEquals(callbackCount.get(), 1);
}