Examples with PlanFragment io.trino.sql.planner.PlanFragment used on opensource projects

Example 36 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class TestLegacyPhasedExecutionSchedule method testJoinWithDeepSources.

@Test
public void testJoinWithDeepSources() {
    PlanFragment buildSourceFragment = createTableScanPlanFragment("buildSource");
    PlanFragment buildMiddleFragment = createExchangePlanFragment("buildMiddle", buildSourceFragment);
    PlanFragment buildTopFragment = createExchangePlanFragment("buildTop", buildMiddleFragment);
    PlanFragment probeSourceFragment = createTableScanPlanFragment("probeSource");
    PlanFragment probeMiddleFragment = createExchangePlanFragment("probeMiddle", probeSourceFragment);
    PlanFragment probeTopFragment = createExchangePlanFragment("probeTop", probeMiddleFragment);
    PlanFragment joinFragment = createJoinPlanFragment(INNER, "join", buildTopFragment, probeTopFragment);
    List<Set<PlanFragmentId>> phases = LegacyPhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildTopFragment, buildMiddleFragment, buildSourceFragment, probeTopFragment, probeMiddleFragment, probeSourceFragment));
    assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildTopFragment.getId()), ImmutableSet.of(buildMiddleFragment.getId()), ImmutableSet.of(buildSourceFragment.getId()), ImmutableSet.of(probeTopFragment.getId()), ImmutableSet.of(probeMiddleFragment.getId()), ImmutableSet.of(probeSourceFragment.getId())));
}

Example 37 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class TestPhasedExecutionSchedule method testAggregation.

@Test
public void testAggregation() {
    PlanFragment sourceFragment = createTableScanPlanFragment("probe");
    PlanFragment aggregationFragment = createAggregationFragment("aggregation", sourceFragment);
    PlanFragment buildFragment = createTableScanPlanFragment("build");
    PlanFragment joinFragment = createJoinPlanFragment(INNER, REPLICATED, "join", buildFragment, aggregationFragment);
    TestingStageExecution sourceStage = new TestingStageExecution(sourceFragment);
    TestingStageExecution aggregationStage = new TestingStageExecution(aggregationFragment);
    TestingStageExecution buildStage = new TestingStageExecution(buildFragment);
    TestingStageExecution joinStage = new TestingStageExecution(joinFragment);
    PhasedExecutionSchedule schedule = PhasedExecutionSchedule.forStages(ImmutableSet.of(sourceStage, aggregationStage, buildStage, joinStage), dynamicFilterService);
    assertThat(schedule.getSortedFragments()).containsExactly(buildFragment.getId(), sourceFragment.getId(), aggregationFragment.getId(), joinFragment.getId());
    // aggregation and source stage should start immediately, join stage should wait for build stage to complete
    DirectedGraph<PlanFragmentId, FragmentsEdge> dependencies = schedule.getFragmentDependency();
    assertThat(dependencies.edgeSet()).containsExactly(new FragmentsEdge(buildFragment.getId(), joinFragment.getId()));
    assertThat(getActiveFragments(schedule)).containsExactly(buildFragment.getId(), sourceFragment.getId(), aggregationFragment.getId());
}

Example 38 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class TestPhasedExecutionSchedule method testDependentStageAbortedBeforeStarted.

@Test
public void testDependentStageAbortedBeforeStarted() {
    PlanFragment sourceFragment = createTableScanPlanFragment("probe");
    PlanFragment aggregationFragment = createAggregationFragment("aggregation", sourceFragment);
    PlanFragment buildFragment = createTableScanPlanFragment("build");
    PlanFragment joinFragment = createJoinPlanFragment(INNER, REPLICATED, "join", buildFragment, aggregationFragment);
    TestingStageExecution sourceStage = new TestingStageExecution(sourceFragment);
    TestingStageExecution aggregationStage = new TestingStageExecution(aggregationFragment);
    TestingStageExecution buildStage = new TestingStageExecution(buildFragment);
    TestingStageExecution joinStage = new TestingStageExecution(joinFragment);
    PhasedExecutionSchedule schedule = PhasedExecutionSchedule.forStages(ImmutableSet.of(sourceStage, aggregationStage, buildStage, joinStage), dynamicFilterService);
    assertThat(schedule.getSortedFragments()).containsExactly(buildFragment.getId(), sourceFragment.getId(), aggregationFragment.getId(), joinFragment.getId());
    // aggregation and source stage should start immediately, join stage should wait for build stage to complete
    DirectedGraph<PlanFragmentId, FragmentsEdge> dependencies = schedule.getFragmentDependency();
    assertThat(dependencies.edgeSet()).containsExactly(new FragmentsEdge(buildFragment.getId(), joinFragment.getId()));
    assertThat(getActiveFragments(schedule)).containsExactly(buildFragment.getId(), sourceFragment.getId(), aggregationFragment.getId());
    // abort non-active join stage
    joinStage.setState(ABORTED);
    // dependencies finish
    buildStage.setState(FINISHED);
    aggregationStage.setState(FINISHED);
    sourceStage.setState(FINISHED);
    // join stage already aborted. Whole schedule should be marked as finished
    schedule.schedule();
    assertThat(schedule.isFinished()).isTrue();
}

Example 39 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class TestPhasedExecutionSchedule method testPartitionedJoin.

@Test
public void testPartitionedJoin() {
    PlanFragment buildFragment = createTableScanPlanFragment("build");
    PlanFragment probeFragment = createTableScanPlanFragment("probe");
    PlanFragment joinFragment = createJoinPlanFragment(INNER, PARTITIONED, "join", buildFragment, probeFragment);
    TestingStageExecution buildStage = new TestingStageExecution(buildFragment);
    TestingStageExecution probeStage = new TestingStageExecution(probeFragment);
    TestingStageExecution joinStage = new TestingStageExecution(joinFragment);
    PhasedExecutionSchedule schedule = PhasedExecutionSchedule.forStages(ImmutableSet.of(buildStage, probeStage, joinStage), dynamicFilterService);
    assertThat(schedule.getSortedFragments()).containsExactly(buildFragment.getId(), probeFragment.getId(), joinFragment.getId());
    // single dependency between build and probe stages
    DirectedGraph<PlanFragmentId, FragmentsEdge> dependencies = schedule.getFragmentDependency();
    assertThat(dependencies.edgeSet()).containsExactlyInAnyOrder(new FragmentsEdge(buildFragment.getId(), probeFragment.getId()));
    // build and join stage should start immediately
    assertThat(getActiveFragments(schedule)).containsExactly(buildFragment.getId(), joinFragment.getId());
    // probe stage should start after build stage is completed
    ListenableFuture<Void> rescheduleFuture = schedule.getRescheduleFuture().orElseThrow();
    assertThat(rescheduleFuture).isNotDone();
    buildStage.setState(FLUSHING);
    assertThat(rescheduleFuture).isDone();
    schedule.schedule();
    assertThat(getActiveFragments(schedule)).containsExactly(joinFragment.getId(), probeFragment.getId());
    // make sure scheduler finishes
    rescheduleFuture = schedule.getRescheduleFuture().orElseThrow();
    assertThat(rescheduleFuture).isNotDone();
    probeStage.setState(FINISHED);
    assertThat(rescheduleFuture).isNotDone();
    joinStage.setState(FINISHED);
    schedule.schedule();
    assertThat(getActiveFragments(schedule)).isEmpty();
    assertThat(schedule.isFinished()).isTrue();
}

Example 40 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, Map<DynamicFilterId, DynamicFilterDomainStats> dynamicFilterDomainStats, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, TypeProvider typeProvider) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, dynamicFilterDomainStats, valuePrinter, fragment.getStatsAndCosts(), planNodeStats).toText(verbose, 1)).append("\n");
    return builder.toString();
}