Search in sources :

Example 21 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class PhasedExecutionSchedule method removeCompletedStage.

private Set<PlanFragmentId> removeCompletedStage(StageExecution stage) {
    // start all stages that depend on completed stage
    PlanFragmentId fragmentId = stage.getFragment().getId();
    if (!fragmentDependency.containsVertex(fragmentId)) {
        // already gone
        return ImmutableSet.of();
    }
    Set<PlanFragmentId> fragmentsToExecute = fragmentDependency.outgoingEdgesOf(fragmentId).stream().map(FragmentsEdge::getTarget).filter(dependentFragmentId -> fragmentDependency.inDegreeOf(dependentFragmentId) == 1).collect(toImmutableSet());
    fragmentDependency.removeVertex(fragmentId);
    fragmentTopology.removeVertex(fragmentId);
    activeStages.remove(stage);
    return fragmentsToExecute;
}
Also used : QueryId(io.trino.spi.QueryId) PlanFragment(io.trino.sql.planner.PlanFragment) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SINGLE(io.trino.sql.planner.plan.AggregationNode.Step.SINGLE) HashMap(java.util.HashMap) SettableFuture(com.google.common.util.concurrent.SettableFuture) PlanNode(io.trino.sql.planner.plan.PlanNode) ArrayList(java.util.ArrayList) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) RUNNING(io.trino.execution.scheduler.StageExecution.State.RUNNING) GuardedBy(oshi.annotation.concurrent.GuardedBy) EdgeFactory(org.jgrapht.EdgeFactory) Verify.verify(com.google.common.base.Verify.verify) Map(java.util.Map) LOCAL(io.trino.sql.planner.plan.ExchangeNode.Scope.LOCAL) Objects.requireNonNull(java.util.Objects.requireNonNull) SpatialJoinNode(io.trino.sql.planner.plan.SpatialJoinNode) AggregationNode(io.trino.sql.planner.plan.AggregationNode) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) JoinNode(io.trino.sql.planner.plan.JoinNode) LinkedHashSet(java.util.LinkedHashSet) DynamicFilterService(io.trino.server.DynamicFilterService) ImmutableSet(com.google.common.collect.ImmutableSet) DefaultDirectedGraph(org.jgrapht.graph.DefaultDirectedGraph) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) IndexJoinNode(io.trino.sql.planner.plan.IndexJoinNode) Set(java.util.Set) SemiJoinNode(io.trino.sql.planner.plan.SemiJoinNode) SCHEDULED(io.trino.execution.scheduler.StageExecution.State.SCHEDULED) StrongConnectivityInspector(org.jgrapht.alg.StrongConnectivityInspector) FLUSHING(io.trino.execution.scheduler.StageExecution.State.FLUSHING) Objects(java.util.Objects) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) State(io.trino.execution.scheduler.StageExecution.State) Ordering(com.google.common.collect.Ordering) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) StageExecution(io.trino.execution.scheduler.StageExecution) ExchangeNode(io.trino.sql.planner.plan.ExchangeNode) DirectedGraph(org.jgrapht.DirectedGraph) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) FINAL(io.trino.sql.planner.plan.AggregationNode.Step.FINAL) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId)

Example 22 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class PipelinedStageExecution method createPipelinedStageExecution.

public static PipelinedStageExecution createPipelinedStageExecution(SqlStage stage, Map<PlanFragmentId, OutputBufferManager> outputBufferManagers, TaskLifecycleListener taskLifecycleListener, FailureDetector failureDetector, Executor executor, Optional<int[]> bucketToPartition, int attempt) {
    PipelinedStageStateMachine stateMachine = new PipelinedStageStateMachine(stage.getStageId(), executor);
    ImmutableMap.Builder<PlanFragmentId, RemoteSourceNode> exchangeSources = ImmutableMap.builder();
    for (RemoteSourceNode remoteSourceNode : stage.getFragment().getRemoteSourceNodes()) {
        for (PlanFragmentId planFragmentId : remoteSourceNode.getSourceFragmentIds()) {
            exchangeSources.put(planFragmentId, remoteSourceNode);
        }
    }
    PipelinedStageExecution execution = new PipelinedStageExecution(stateMachine, stage, outputBufferManagers, taskLifecycleListener, failureDetector, executor, bucketToPartition, exchangeSources.buildOrThrow(), attempt);
    execution.initialize();
    return execution;
}
Also used : RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) ImmutableMap(com.google.common.collect.ImmutableMap)

Aggregations

PlanFragmentId (io.trino.sql.planner.plan.PlanFragmentId)22 PlanFragment (io.trino.sql.planner.PlanFragment)15 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)10 RemoteSourceNode (io.trino.sql.planner.plan.RemoteSourceNode)10 Test (org.testng.annotations.Test)7 PartitioningScheme (io.trino.sql.planner.PartitioningScheme)6 Symbol (io.trino.sql.planner.Symbol)6 ImmutableMap (com.google.common.collect.ImmutableMap)5 ImmutableSet (com.google.common.collect.ImmutableSet)5 FragmentsEdge (io.trino.execution.scheduler.policy.PhasedExecutionSchedule.FragmentsEdge)5 PlanUtils.createBroadcastAndPartitionedJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastAndPartitionedJoinPlanFragment)5 PlanUtils.createBroadcastJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastJoinPlanFragment)5 PlanUtils.createJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createJoinPlanFragment)5 PlanUtils.createTableScanPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createTableScanPlanFragment)5 JoinNode (io.trino.sql.planner.plan.JoinNode)5 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)4 ExchangeSourceHandle (io.trino.spi.exchange.ExchangeSourceHandle)4 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)3