Search in sources :

Example 1 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!started.get() || !needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // currentRequest is always cleared when request is complete
    if (currentRequest != null) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<Void> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<SplitAssignment> splitAssignments = getSplitAssignments();
    VersionedDynamicFilterDomains dynamicFilterDomains = outboundDynamicFiltersCollector.acknowledgeAndGetNewDomains(sentDynamicFiltersVersion);
    // Workers don't need the embedded JSON representation when the fragment is sent
    Optional<PlanFragment> fragment = sendPlan.get() ? Optional.of(planFragment.withoutEmbeddedJsonRepresentation()) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, splitAssignments, outputBuffers.get(), dynamicFilterDomains.getDynamicFilterDomains());
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toJsonBytes(updateRequest);
    if (fragment.isPresent()) {
        stats.updateWithPlanBytes(taskUpdateRequestJson.length);
    }
    if (!dynamicFilterDomains.getDynamicFilterDomains().isEmpty()) {
        stats.updateWithDynamicFilterBytes(taskUpdateRequestJson.length);
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = preparePost().setUri(uriBuilder.build()).setHeader(HttpHeaders.CONTENT_TYPE, MediaType.JSON_UTF_8.toString()).setBodyGenerator(createStaticBodyGenerator(taskUpdateRequestJson)).build();
    updateErrorTracker.startRequest();
    ListenableFuture<JsonResponse<TaskInfo>> future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskInfoCodec));
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(splitAssignments, dynamicFilterDomains.getVersion()), request.getUri(), stats), executor);
}
Also used : TaskUpdateRequest(io.trino.server.TaskUpdateRequest) TaskUpdateRequest(io.trino.server.TaskUpdateRequest) FailTaskRequest(io.trino.server.FailTaskRequest) Request(io.airlift.http.client.Request) SplitAssignment(io.trino.execution.SplitAssignment) TaskStatus(io.trino.execution.TaskStatus) PlanFragment(io.trino.sql.planner.PlanFragment) JsonResponse(io.airlift.http.client.FullJsonResponseHandler.JsonResponse) VersionedDynamicFilterDomains(io.trino.execution.DynamicFiltersCollector.VersionedDynamicFilterDomains) HttpUriBuilder(io.airlift.http.client.HttpUriBuilder)

Example 2 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.

@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
    // determine output fragment
    Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
    Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
    Visitor visitor = new Visitor(fragments);
    rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId()));
    return visitor.getSchedulerOrder();
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) UnionNode(io.trino.sql.planner.plan.UnionNode) PlanNode(io.trino.sql.planner.plan.PlanNode) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) RUNNING(io.trino.execution.scheduler.StageExecution.State.RUNNING) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) SpatialJoinNode(io.trino.sql.planner.plan.SpatialJoinNode) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) JoinNode(io.trino.sql.planner.plan.JoinNode) LinkedHashSet(java.util.LinkedHashSet) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) IndexJoinNode(io.trino.sql.planner.plan.IndexJoinNode) Set(java.util.Set) SemiJoinNode(io.trino.sql.planner.plan.SemiJoinNode) SCHEDULED(io.trino.execution.scheduler.StageExecution.State.SCHEDULED) FLUSHING(io.trino.execution.scheduler.StageExecution.State.FLUSHING) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Ordering(com.google.common.collect.Ordering) Function.identity(java.util.function.Function.identity) StageExecution(io.trino.execution.scheduler.StageExecution) ExchangeNode(io.trino.sql.planner.plan.ExchangeNode) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) PlanFragment(io.trino.sql.planner.PlanFragment) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 3 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class LegacyPhasedExecutionSchedule method extractPhases.

@VisibleForTesting
static List<Set<PlanFragmentId>> extractPhases(Collection<PlanFragment> fragments) {
    // Build a graph where the plan fragments are vertexes and the edges represent
    // a before -> after relationship.  For example, a join hash build has an edge
    // to the join probe.
    DirectedGraph<PlanFragmentId, DefaultEdge> graph = new DefaultDirectedGraph<>(DefaultEdge.class);
    fragments.forEach(fragment -> graph.addVertex(fragment.getId()));
    Visitor visitor = new Visitor(fragments, graph);
    for (PlanFragment fragment : fragments) {
        visitor.processFragment(fragment.getId());
    }
    // Computes all the strongly connected components of the directed graph.
    // These are the "phases" which hold the set of fragments that must be started
    // at the same time to avoid deadlock.
    List<Set<PlanFragmentId>> components = new StrongConnectivityInspector<>(graph).stronglyConnectedSets();
    Map<PlanFragmentId, Set<PlanFragmentId>> componentMembership = new HashMap<>();
    for (Set<PlanFragmentId> component : components) {
        for (PlanFragmentId planFragmentId : component) {
            componentMembership.put(planFragmentId, component);
        }
    }
    // build graph of components (phases)
    DirectedGraph<Set<PlanFragmentId>, DefaultEdge> componentGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
    components.forEach(componentGraph::addVertex);
    for (DefaultEdge edge : graph.edgeSet()) {
        PlanFragmentId source = graph.getEdgeSource(edge);
        PlanFragmentId target = graph.getEdgeTarget(edge);
        Set<PlanFragmentId> from = componentMembership.get(source);
        Set<PlanFragmentId> to = componentMembership.get(target);
        if (!from.equals(to)) {
            // the topological order iterator below doesn't include vertices that have self-edges, so don't add them
            componentGraph.addEdge(from, to);
        }
    }
    List<Set<PlanFragmentId>> schedulePhases = ImmutableList.copyOf(new TopologicalOrderIterator<>(componentGraph));
    return schedulePhases;
}
Also used : HashSet(java.util.HashSet) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) DefaultDirectedGraph(org.jgrapht.graph.DefaultDirectedGraph) HashMap(java.util.HashMap) DefaultEdge(org.jgrapht.graph.DefaultEdge) PlanFragment(io.trino.sql.planner.PlanFragment) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 4 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class PhasedExecutionSchedule method extractDependenciesAndReturnNonLazyFragments.

private Set<PlanFragmentId> extractDependenciesAndReturnNonLazyFragments(Collection<StageExecution> stages) {
    if (stages.isEmpty()) {
        return ImmutableSet.of();
    }
    QueryId queryId = stages.stream().map(stage -> stage.getStageId().getQueryId()).findAny().orElseThrow();
    List<PlanFragment> fragments = stages.stream().map(StageExecution::getFragment).collect(toImmutableList());
    // Build a graph where the plan fragments are vertexes and the edges represent
    // a before -> after relationship. Destination fragment should be started only
    // when source fragment is completed. For example, a join hash build has an edge
    // to the join probe.
    Visitor visitor = new Visitor(queryId, fragments);
    visitor.processAllFragments();
    // Make sure there are no strongly connected components as it would mean circular dependency between stages
    List<Set<PlanFragmentId>> components = new StrongConnectivityInspector<>(fragmentDependency).stronglyConnectedSets();
    verify(components.size() == fragmentDependency.vertexSet().size(), "circular dependency between stages");
    return visitor.getNonLazyFragments();
}
Also used : ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) LinkedHashSet(java.util.LinkedHashSet) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) QueryId(io.trino.spi.QueryId) PlanFragment(io.trino.sql.planner.PlanFragment)

Example 5 with PlanFragment

use of io.trino.sql.planner.PlanFragment in project trino by trinodb.

the class TestSqlStage method createExchangePlanFragment.

private static PlanFragment createExchangePlanFragment() {
    PlanNode planNode = new RemoteSourceNode(new PlanNodeId("exchange"), ImmutableList.of(new PlanFragmentId("source")), ImmutableList.of(new Symbol("column")), Optional.empty(), REPARTITION, RetryPolicy.NONE);
    ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder();
    for (Symbol symbol : planNode.getOutputSymbols()) {
        types.put(symbol, VARCHAR);
    }
    return new PlanFragment(new PlanFragmentId("exchange_fragment_id"), planNode, types.buildOrThrow(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty());
}
Also used : PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Type(io.trino.spi.type.Type) PlanNode(io.trino.sql.planner.plan.PlanNode) Symbol(io.trino.sql.planner.Symbol) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) ImmutableMap(com.google.common.collect.ImmutableMap) PlanFragment(io.trino.sql.planner.PlanFragment)

Aggregations

PlanFragment (io.trino.sql.planner.PlanFragment)41 Test (org.testng.annotations.Test)22 PlanFragmentId (io.trino.sql.planner.plan.PlanFragmentId)16 ImmutableSet (com.google.common.collect.ImmutableSet)12 NodeTaskMap (io.trino.execution.NodeTaskMap)11 PipelinedStageExecution.createPipelinedStageExecution (io.trino.execution.scheduler.PipelinedStageExecution.createPipelinedStageExecution)11 SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler (io.trino.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler)11 PlanUtils.createBroadcastJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastJoinPlanFragment)11 PlanUtils.createJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createJoinPlanFragment)11 PlanUtils.createTableScanPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createTableScanPlanFragment)11 Set (java.util.Set)11 Symbol (io.trino.sql.planner.Symbol)10 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)10 PartitioningScheme (io.trino.sql.planner.PartitioningScheme)9 MockRemoteTask (io.trino.execution.MockRemoteTaskFactory.MockRemoteTask)8 PartitionedSplitsInfo (io.trino.execution.PartitionedSplitsInfo)8 RemoteTask (io.trino.execution.RemoteTask)8 RemoteSourceNode (io.trino.sql.planner.plan.RemoteSourceNode)8 JoinNode (io.trino.sql.planner.plan.JoinNode)7 Duration (io.airlift.units.Duration)6