Examples with DynamicFilterDomainStats io.trino.server.DynamicFilterService.DynamicFilterDomainStats used on opensource projects

Example 11 with DynamicFilterDomainStats

use of io.trino.server.DynamicFilterService.DynamicFilterDomainStats in project trino by trinodb.

the class BaseDynamicPartitionPruningTest method testRightJoinWithNonSelectiveBuildSide.

@Test(timeOut = 30_000)
public void testRightJoinWithNonSelectiveBuildSide() {
    @Language("SQL") String selectQuery = "SELECT * FROM partitioned_lineitem l RIGHT JOIN supplier s ON l.suppkey = s.suppkey";
    ResultWithQueryId<MaterializedResult> result = getDistributedQueryRunner().executeWithQueryId(getSession(), selectQuery);
    MaterializedResult expected = computeActual(withDynamicFilteringDisabled(), selectQuery);
    assertEqualsIgnoreOrder(result.getResult(), expected);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), getQualifiedTableName(PARTITIONED_LINEITEM));
    // Probe-side is fully scanned
    assertEquals(probeStats.getInputPositions(), LINEITEM_COUNT);
    DynamicFiltersStats dynamicFiltersStats = getDynamicFilteringStats(result.getQueryId());
    assertEquals(dynamicFiltersStats.getTotalDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getLazyDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getReplicatedDynamicFilters(), 0L);
    assertEquals(dynamicFiltersStats.getDynamicFiltersCompleted(), 1L);
    DynamicFilterDomainStats domainStats = getOnlyElement(dynamicFiltersStats.getDynamicFilterDomainStats());
    assertThat(domainStats.getSimplifiedDomain()).isEqualTo(getSimplifiedDomainString(1L, 100L, 100, BIGINT));
}

Example 12 with DynamicFilterDomainStats

use of io.trino.server.DynamicFilterService.DynamicFilterDomainStats in project trino by trinodb.

the class BaseFailureRecoveryTest method testJoinDynamicFilteringEnabled.

@Test(invocationCount = INVOCATION_COUNT)
public void testJoinDynamicFilteringEnabled() {
    @Language("SQL") String selectQuery = "SELECT * FROM partitioned_lineitem JOIN supplier ON partitioned_lineitem.suppkey = supplier.suppkey " + "AND supplier.name = 'Supplier#000000001'";
    testSelect(selectQuery, Optional.of(enableDynamicFiltering(true)), queryId -> {
        DynamicFiltersStats dynamicFiltersStats = getDynamicFilteringStats(queryId);
        assertThat(dynamicFiltersStats.getLazyDynamicFilters()).isEqualTo(1);
        DynamicFilterDomainStats domainStats = getOnlyElement(dynamicFiltersStats.getDynamicFilterDomainStats());
        assertThat(domainStats.getSimplifiedDomain()).isEqualTo(singleValue(BIGINT, 1L).toString(getSession().toConnectorSession()));
        OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(queryId, getQualifiedTableName(PARTITIONED_LINEITEM));
        // Currently, stats from all attempts are combined.
        // Asserting on multiple of 615L as well in case the probe scan was completed twice
        assertThat(probeStats.getInputPositions()).isIn(615L, 1230L);
    });
}

Example 13 with DynamicFilterDomainStats

use of io.trino.server.DynamicFilterService.DynamicFilterDomainStats in project trino by trinodb.

the class TestDynamicFilterService method testReplicatedDynamicFilter.

@Test
public void testReplicatedDynamicFilter() {
    DynamicFilterService dynamicFilterService = createDynamicFilterService();
    DynamicFilterId filterId1 = new DynamicFilterId("df1");
    SymbolAllocator symbolAllocator = new SymbolAllocator();
    Symbol symbol1 = symbolAllocator.newSymbol("DF_SYMBOL1", INTEGER);
    Expression df1 = symbol1.toSymbolReference();
    QueryId queryId = new QueryId("query");
    StageId stageId1 = new StageId(queryId, 1);
    dynamicFilterService.registerQuery(queryId, session, ImmutableSet.of(filterId1), ImmutableSet.of(), ImmutableSet.of(filterId1));
    DynamicFilter dynamicFilter = dynamicFilterService.createDynamicFilter(queryId, ImmutableList.of(new DynamicFilters.Descriptor(filterId1, df1)), ImmutableMap.of(symbol1, new TestingColumnHandle("probeColumnA")), symbolAllocator.getTypes());
    assertEquals(dynamicFilter.getColumnsCovered(), Set.of(new TestingColumnHandle("probeColumnA")), "columns covered");
    assertTrue(dynamicFilter.getCurrentPredicate().isAll());
    // assert initial dynamic filtering stats
    DynamicFiltersStats stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getTotalDynamicFilters(), 1);
    assertEquals(stats.getDynamicFiltersCompleted(), 0);
    assertEquals(stats.getReplicatedDynamicFilters(), 1);
    assertEquals(stats.getLazyDynamicFilters(), 0);
    // filterId1 wasn't marked as lazy
    assertFalse(dynamicFilter.isComplete());
    assertFalse(dynamicFilter.isAwaitable());
    assertTrue(dynamicFilter.isBlocked().isDone());
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId1, 0, 0), ImmutableMap.of(filterId1, singleValue(INTEGER, 1L)));
    // tuple domain from single broadcast join task is sufficient
    assertEquals(dynamicFilter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(new TestingColumnHandle("probeColumnA"), singleValue(INTEGER, 1L))));
    assertTrue(dynamicFilter.isComplete());
    assertFalse(dynamicFilter.isAwaitable());
    stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getTotalDynamicFilters(), 1);
    assertEquals(stats.getDynamicFiltersCompleted(), 1);
    assertEquals(stats.getReplicatedDynamicFilters(), 1);
    assertEquals(stats.getLazyDynamicFilters(), 0);
    assertEquals(stats.getDynamicFilterDomainStats(), ImmutableList.of(new DynamicFilterDomainStats(filterId1, Domain.singleValue(INTEGER, 1L).toString(session.toConnectorSession()))));
}

Example 14 with DynamicFilterDomainStats

use of io.trino.server.DynamicFilterService.DynamicFilterDomainStats in project trino by trinodb.

the class TestDynamicFilterService method testDynamicFilterSummaryCompletion.

@Test
public void testDynamicFilterSummaryCompletion() {
    DynamicFilterService dynamicFilterService = createDynamicFilterService();
    DynamicFilterId filterId = new DynamicFilterId("df");
    QueryId queryId = new QueryId("query");
    StageId stageId = new StageId(queryId, 0);
    dynamicFilterService.registerQuery(queryId, session, ImmutableSet.of(filterId), ImmutableSet.of(filterId), ImmutableSet.of());
    dynamicFilterService.stageCannotScheduleMoreTasks(stageId, 0, 3);
    assertFalse(dynamicFilterService.getSummary(queryId, filterId).isPresent());
    // assert initial dynamic filtering stats
    DynamicFiltersStats stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getTotalDynamicFilters(), 1);
    assertEquals(stats.getDynamicFiltersCompleted(), 0);
    assertEquals(stats.getLazyDynamicFilters(), 1);
    assertEquals(stats.getReplicatedDynamicFilters(), 0);
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId, 0, 0), ImmutableMap.of(filterId, singleValue(INTEGER, 1L)));
    assertFalse(dynamicFilterService.getSummary(queryId, filterId).isPresent());
    stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getDynamicFiltersCompleted(), 0);
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId, 1, 0), ImmutableMap.of(filterId, singleValue(INTEGER, 2L)));
    assertFalse(dynamicFilterService.getSummary(queryId, filterId).isPresent());
    stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getDynamicFiltersCompleted(), 0);
    dynamicFilterService.addTaskDynamicFilters(new TaskId(stageId, 2, 0), ImmutableMap.of(filterId, singleValue(INTEGER, 3L)));
    Optional<Domain> summary = dynamicFilterService.getSummary(queryId, filterId);
    assertTrue(summary.isPresent());
    assertEquals(summary.get(), multipleValues(INTEGER, ImmutableList.of(1L, 2L, 3L)));
    stats = dynamicFilterService.getDynamicFilteringStats(queryId, session);
    assertEquals(stats.getDynamicFiltersCompleted(), 1);
    assertEquals(stats.getLazyDynamicFilters(), 1);
    assertEquals(stats.getReplicatedDynamicFilters(), 0);
    assertEquals(stats.getDynamicFilterDomainStats(), ImmutableList.of(new DynamicFilterDomainStats(filterId, getSimplifiedDomainString(1L, 3L, 3, INTEGER))));
}

Example 15 with DynamicFilterDomainStats

use of io.trino.server.DynamicFilterService.DynamicFilterDomainStats in project trino by trinodb.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, Map<DynamicFilterId, DynamicFilterDomainStats> dynamicFilterDomainStats, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, TypeProvider typeProvider) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, dynamicFilterDomainStats, valuePrinter, fragment.getStatsAndCosts(), planNodeStats).toText(verbose, 1)).append("\n");
    return builder.toString();
}