Examples with DynamicFilterId io.trino.sql.planner.plan.DynamicFilterId used on opensource projects

Example 56 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, Map<DynamicFilterId, DynamicFilterDomainStats> dynamicFilterDomainStats, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, TypeProvider typeProvider) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, dynamicFilterDomainStats, valuePrinter, fragment.getStatsAndCosts(), planNodeStats).toText(verbose, 1)).append("\n");
    return builder.toString();
}

Example 57 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFilterConsumer method testShortCircuitOnAllTupleDomain.

@Test
public void testShortCircuitOnAllTupleDomain() throws Exception {
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(ImmutableMap.of(new DynamicFilterId("123"), 0), ImmutableMap.of(new DynamicFilterId("123"), INTEGER), 2);
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<DynamicFilterId, Domain>> result = filter.getDynamicFilterDomains();
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.all(INTEGER))));
    assertEquals(result.get(), ImmutableMap.of(new DynamicFilterId("123"), Domain.all(INTEGER)));
    // adding another partition domain won't change final domain
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 1L))));
    assertEquals(result.get(), ImmutableMap.of(new DynamicFilterId("123"), Domain.all(INTEGER)));
}

Example 58 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFiltersCollector method testDynamicFilterCancellation.

@Test
public void testDynamicFilterCancellation() {
    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(TEST_SESSION);
    DynamicFilterId filterId = new DynamicFilterId("filter");
    collector.register(ImmutableSet.of(filterId));
    SymbolAllocator symbolAllocator = new SymbolAllocator();
    Symbol symbol = symbolAllocator.newSymbol("symbol", BIGINT);
    ColumnHandle column = new TestingColumnHandle("column");
    DynamicFilter filter = createDynamicFilter(collector, ImmutableList.of(new DynamicFilters.Descriptor(filterId, symbol.toSymbolReference())), ImmutableMap.of(symbol, column), symbolAllocator.getTypes());
    // Filter is blocked and not completed.
    CompletableFuture<?> isBlocked = filter.isBlocked();
    assertFalse(filter.isComplete());
    assertFalse(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.all());
    // DynamicFilter future cancellation should not affect LocalDynamicFiltersCollector
    assertFalse(isBlocked.cancel(false));
    assertFalse(isBlocked.isDone());
    assertFalse(filter.isComplete());
    Domain domain = Domain.singleValue(BIGINT, 7L);
    collector.collectDynamicFilterDomains(ImmutableMap.of(filterId, domain));
    // Unblocked and completed.
    assertTrue(filter.isComplete());
    assertTrue(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)));
}

Example 59 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFiltersCollector method testMultipleProbeColumns.

@Test
public void testMultipleProbeColumns() {
    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(TEST_SESSION);
    DynamicFilterId filterId = new DynamicFilterId("filter");
    collector.register(ImmutableSet.of(filterId));
    // Same build-side column being matched to multiple probe-side columns.
    SymbolAllocator symbolAllocator = new SymbolAllocator();
    Symbol symbol1 = symbolAllocator.newSymbol("symbol1", BIGINT);
    Symbol symbol2 = symbolAllocator.newSymbol("symbol2", BIGINT);
    ColumnHandle column1 = new TestingColumnHandle("column1");
    ColumnHandle column2 = new TestingColumnHandle("column2");
    DynamicFilter filter = createDynamicFilter(collector, ImmutableList.of(new DynamicFilters.Descriptor(filterId, symbol1.toSymbolReference()), new DynamicFilters.Descriptor(filterId, symbol2.toSymbolReference())), ImmutableMap.of(symbol1, column1, symbol2, column2), symbolAllocator.getTypes());
    assertEquals(filter.getColumnsCovered(), Set.of(column1, column2), "columns covered");
    // Filter is blocked and not completed.
    CompletableFuture<?> isBlocked = filter.isBlocked();
    assertFalse(filter.isComplete());
    assertTrue(filter.isAwaitable());
    assertFalse(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.all());
    Domain domain = Domain.singleValue(BIGINT, 7L);
    collector.collectDynamicFilterDomains(ImmutableMap.of(filterId, domain));
    // Unblocked and completed.
    assertTrue(filter.isComplete());
    assertFalse(filter.isAwaitable());
    assertTrue(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(column1, domain, column2, domain)));
}

Example 60 with DynamicFilterId

use of io.trino.sql.planner.plan.DynamicFilterId in project trino by trinodb.

the class TestLocalDynamicFiltersCollector method testUnusedDynamicFilter.

@Test
public void testUnusedDynamicFilter() {
    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(TEST_SESSION);
    DynamicFilterId unusedFilterId = new DynamicFilterId("unused");
    DynamicFilterId usedFilterId = new DynamicFilterId("used");
    collector.register(ImmutableSet.of(unusedFilterId));
    collector.register(ImmutableSet.of(usedFilterId));
    // One of the dynamic filters is not used for the the table scan.
    SymbolAllocator symbolAllocator = new SymbolAllocator();
    Symbol usedSymbol = symbolAllocator.newSymbol("used", BIGINT);
    ColumnHandle usedColumn = new TestingColumnHandle("used");
    DynamicFilter filter = createDynamicFilter(collector, ImmutableList.of(new DynamicFilters.Descriptor(usedFilterId, usedSymbol.toSymbolReference())), ImmutableMap.of(usedSymbol, usedColumn), symbolAllocator.getTypes());
    // Filter is blocking and not completed.
    CompletableFuture<?> isBlocked = filter.isBlocked();
    assertFalse(filter.isComplete());
    assertTrue(filter.isAwaitable());
    assertFalse(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.all());
    collector.collectDynamicFilterDomains(ImmutableMap.of(unusedFilterId, Domain.singleValue(BIGINT, 1L)));
    // This dynamic filter is unused here - has no effect on blocking/completion of the above future.
    assertFalse(filter.isComplete());
    assertFalse(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.all());
    collector.collectDynamicFilterDomains(ImmutableMap.of(usedFilterId, Domain.singleValue(BIGINT, 2L)));
    // Unblocked and completed.
    assertTrue(filter.isComplete());
    assertFalse(filter.isAwaitable());
    assertTrue(isBlocked.isDone());
    assertEquals(filter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(usedColumn, Domain.singleValue(BIGINT, 2L))));
}