Search in sources :

Example 1 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class FaultTolerantStageScheduler method schedule.

public synchronized void schedule() throws Exception {
    if (failure != null) {
        propagateIfPossible(failure, Exception.class);
        throw new RuntimeException(failure);
    }
    if (closed) {
        return;
    }
    if (isFinished()) {
        return;
    }
    if (!blocked.isDone()) {
        return;
    }
    if (taskSource == null) {
        Map<PlanFragmentId, ListenableFuture<List<ExchangeSourceHandle>>> sourceHandles = sourceExchanges.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> toListenableFuture(entry.getValue().getSourceHandles())));
        List<ListenableFuture<List<ExchangeSourceHandle>>> blockedFutures = sourceHandles.values().stream().filter(future -> !future.isDone()).collect(toImmutableList());
        if (!blockedFutures.isEmpty()) {
            blocked = asVoid(allAsList(blockedFutures));
            return;
        }
        Multimap<PlanFragmentId, ExchangeSourceHandle> exchangeSources = sourceHandles.entrySet().stream().collect(flatteningToImmutableListMultimap(Map.Entry::getKey, entry -> getFutureValue(entry.getValue()).stream()));
        taskSource = taskSourceFactory.create(session, stage.getFragment(), sourceExchanges, exchangeSources, stage::recordGetSplitTime, sourceBucketToPartitionMap, sourceBucketNodeMap);
    }
    while (!queuedPartitions.isEmpty() || !taskSource.isFinished()) {
        while (queuedPartitions.isEmpty() && !taskSource.isFinished()) {
            List<TaskDescriptor> tasks = taskSource.getMoreTasks();
            for (TaskDescriptor task : tasks) {
                queuedPartitions.add(task.getPartitionId());
                allPartitions.add(task.getPartitionId());
                taskDescriptorStorage.put(stage.getStageId(), task);
                sinkExchange.ifPresent(exchange -> {
                    ExchangeSinkHandle exchangeSinkHandle = exchange.addSink(task.getPartitionId());
                    partitionToExchangeSinkHandleMap.put(task.getPartitionId(), exchangeSinkHandle);
                });
            }
            if (taskSource.isFinished()) {
                sinkExchange.ifPresent(Exchange::noMoreSinks);
            }
        }
        if (queuedPartitions.isEmpty()) {
            break;
        }
        int partition = queuedPartitions.peek();
        Optional<TaskDescriptor> taskDescriptorOptional = taskDescriptorStorage.get(stage.getStageId(), partition);
        if (taskDescriptorOptional.isEmpty()) {
            // query has been terminated
            return;
        }
        TaskDescriptor taskDescriptor = taskDescriptorOptional.get();
        MemoryRequirements memoryRequirements = partitionMemoryRequirements.computeIfAbsent(partition, ignored -> partitionMemoryEstimator.getInitialMemoryRequirements(session, taskDescriptor.getNodeRequirements().getMemory()));
        if (nodeLease == null) {
            NodeRequirements nodeRequirements = taskDescriptor.getNodeRequirements();
            nodeRequirements = nodeRequirements.withMemory(memoryRequirements.getRequiredMemory());
            nodeLease = nodeAllocator.acquire(nodeRequirements);
        }
        if (!nodeLease.getNode().isDone()) {
            blocked = asVoid(nodeLease.getNode());
            return;
        }
        NodeInfo node = getFutureValue(nodeLease.getNode());
        queuedPartitions.poll();
        Multimap<PlanNodeId, Split> tableScanSplits = taskDescriptor.getSplits();
        Multimap<PlanNodeId, Split> remoteSplits = createRemoteSplits(taskDescriptor.getExchangeSourceHandles());
        Multimap<PlanNodeId, Split> taskSplits = ImmutableListMultimap.<PlanNodeId, Split>builder().putAll(tableScanSplits).putAll(remoteSplits).build();
        int attemptId = getNextAttemptIdForPartition(partition);
        OutputBuffers outputBuffers;
        Optional<ExchangeSinkInstanceHandle> exchangeSinkInstanceHandle;
        if (sinkExchange.isPresent()) {
            ExchangeSinkHandle sinkHandle = partitionToExchangeSinkHandleMap.get(partition);
            exchangeSinkInstanceHandle = Optional.of(sinkExchange.get().instantiateSink(sinkHandle, attemptId));
            outputBuffers = createSpoolingExchangeOutputBuffers(exchangeSinkInstanceHandle.get());
        } else {
            exchangeSinkInstanceHandle = Optional.empty();
            // stage will be consumed by the coordinator using direct exchange
            outputBuffers = createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(new OutputBuffers.OutputBufferId(0), 0).withNoMoreBufferIds();
        }
        Set<PlanNodeId> allSourcePlanNodeIds = ImmutableSet.<PlanNodeId>builder().addAll(stage.getFragment().getPartitionedSources()).addAll(stage.getFragment().getRemoteSourceNodes().stream().map(RemoteSourceNode::getId).iterator()).build();
        RemoteTask task = stage.createTask(node.getNode(), partition, attemptId, sinkBucketToPartitionMap, outputBuffers, taskSplits, allSourcePlanNodeIds.stream().collect(toImmutableListMultimap(Function.identity(), planNodeId -> Lifespan.taskWide())), allSourcePlanNodeIds).orElseThrow(() -> new VerifyException("stage execution is expected to be active"));
        partitionToRemoteTaskMap.put(partition, task);
        runningTasks.put(task.getTaskId(), task);
        runningNodes.put(task.getTaskId(), nodeLease);
        nodeLease = null;
        if (taskFinishedFuture == null) {
            taskFinishedFuture = SettableFuture.create();
        }
        taskLifecycleListener.taskCreated(stage.getFragment().getId(), task);
        task.addStateChangeListener(taskStatus -> updateTaskStatus(taskStatus, exchangeSinkInstanceHandle));
        task.start();
    }
    if (taskFinishedFuture != null && !taskFinishedFuture.isDone()) {
        blocked = taskFinishedFuture;
    }
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) SettableFuture(com.google.common.util.concurrent.SettableFuture) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Throwables.propagateIfPossible(com.google.common.base.Throwables.propagateIfPossible) ImmutableListMultimap.toImmutableListMultimap(com.google.common.collect.ImmutableListMultimap.toImmutableListMultimap) MemoryRequirements(io.trino.execution.scheduler.PartitionMemoryEstimator.MemoryRequirements) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) Map(java.util.Map) SpoolingExchangeInput(io.trino.split.RemoteSplit.SpoolingExchangeInput) REMOTE_HOST_GONE(io.trino.spi.StandardErrorCode.REMOTE_HOST_GONE) Futures.immediateVoidFuture(com.google.common.util.concurrent.Futures.immediateVoidFuture) ImmutableSet(com.google.common.collect.ImmutableSet) ExchangeSinkInstanceHandle(io.trino.spi.exchange.ExchangeSinkInstanceHandle) OutputBuffers.createSpoolingExchangeOutputBuffers(io.trino.execution.buffer.OutputBuffers.createSpoolingExchangeOutputBuffers) ImmutableMap(com.google.common.collect.ImmutableMap) ExecutionFailureInfo(io.trino.execution.ExecutionFailureInfo) Futures.allAsList(com.google.common.util.concurrent.Futures.allAsList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) MoreFutures.toListenableFuture(io.airlift.concurrent.MoreFutures.toListenableFuture) Set(java.util.Set) TrinoException(io.trino.spi.TrinoException) GONE(io.trino.failuredetector.FailureDetector.State.GONE) GuardedBy(javax.annotation.concurrent.GuardedBy) TaskId(io.trino.execution.TaskId) ExchangeSinkHandle(io.trino.spi.exchange.ExchangeSinkHandle) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Split(io.trino.metadata.Split) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Optional(java.util.Optional) Queue(java.util.Queue) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) OutputBuffers.createInitialEmptyOutputBuffers(io.trino.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) ExchangeSourceHandle(io.trino.spi.exchange.ExchangeSourceHandle) Session(io.trino.Session) ImmutableListMultimap.flatteningToImmutableListMultimap(com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) StageId(io.trino.execution.StageId) Logger(io.airlift.log.Logger) HashMap(java.util.HashMap) Multimap(com.google.common.collect.Multimap) ErrorCode(io.trino.spi.ErrorCode) Function(java.util.function.Function) Failures.toFailure(io.trino.util.Failures.toFailure) RemoteSplit(io.trino.split.RemoteSplit) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) USER_ERROR(io.trino.spi.ErrorType.USER_ERROR) Objects.requireNonNull(java.util.Objects.requireNonNull) TaskState(io.trino.execution.TaskState) Lifespan(io.trino.execution.Lifespan) Exchange(io.trino.spi.exchange.Exchange) VerifyException(com.google.common.base.VerifyException) SqlStage(io.trino.execution.SqlStage) FailureDetector(io.trino.failuredetector.FailureDetector) RemoteTask(io.trino.execution.RemoteTask) TaskStatus(io.trino.execution.TaskStatus) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) GENERIC_INTERNAL_ERROR(io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) MoreFutures.asVoid(io.airlift.concurrent.MoreFutures.asVoid) PARTITIONED(io.trino.execution.buffer.OutputBuffers.BufferType.PARTITIONED) Futures.nonCancellationPropagating(com.google.common.util.concurrent.Futures.nonCancellationPropagating) OutputBuffers(io.trino.execution.buffer.OutputBuffers) ArrayDeque(java.util.ArrayDeque) REMOTE_CONNECTOR_ID(io.trino.operator.ExchangeOperator.REMOTE_CONNECTOR_ID) MemoryRequirements(io.trino.execution.scheduler.PartitionMemoryEstimator.MemoryRequirements) ExchangeSourceHandle(io.trino.spi.exchange.ExchangeSourceHandle) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) OutputBuffers.createSpoolingExchangeOutputBuffers(io.trino.execution.buffer.OutputBuffers.createSpoolingExchangeOutputBuffers) OutputBuffers.createInitialEmptyOutputBuffers(io.trino.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers(io.trino.execution.buffer.OutputBuffers) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) ExchangeSinkInstanceHandle(io.trino.spi.exchange.ExchangeSinkInstanceHandle) RemoteTask(io.trino.execution.RemoteTask) Exchange(io.trino.spi.exchange.Exchange) ExchangeSinkHandle(io.trino.spi.exchange.ExchangeSinkHandle) VerifyException(com.google.common.base.VerifyException) MoreFutures.toListenableFuture(io.airlift.concurrent.MoreFutures.toListenableFuture) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Split(io.trino.metadata.Split) RemoteSplit(io.trino.split.RemoteSplit) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) HashMap(java.util.HashMap)

Example 2 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class PipelinedStageExecution method updateSourceTasksOutputBuffers.

private synchronized void updateSourceTasksOutputBuffers(Consumer<OutputBufferManager> updater) {
    for (PlanFragmentId sourceFragment : exchangeSources.keySet()) {
        OutputBufferManager outputBufferManager = outputBufferManagers.get(sourceFragment);
        updater.accept(outputBufferManager);
        for (RemoteTask sourceTask : sourceTasks.get(sourceFragment)) {
            sourceTask.setOutputBuffers(outputBufferManager.getOutputBuffers());
        }
    }
}
Also used : RemoteTask(io.trino.execution.RemoteTask) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId)

Example 3 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.

@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
    // determine output fragment
    Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
    Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
    Visitor visitor = new Visitor(fragments);
    rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId()));
    return visitor.getSchedulerOrder();
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) UnionNode(io.trino.sql.planner.plan.UnionNode) PlanNode(io.trino.sql.planner.plan.PlanNode) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) RUNNING(io.trino.execution.scheduler.StageExecution.State.RUNNING) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) SpatialJoinNode(io.trino.sql.planner.plan.SpatialJoinNode) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) JoinNode(io.trino.sql.planner.plan.JoinNode) LinkedHashSet(java.util.LinkedHashSet) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) IndexJoinNode(io.trino.sql.planner.plan.IndexJoinNode) Set(java.util.Set) SemiJoinNode(io.trino.sql.planner.plan.SemiJoinNode) SCHEDULED(io.trino.execution.scheduler.StageExecution.State.SCHEDULED) FLUSHING(io.trino.execution.scheduler.StageExecution.State.FLUSHING) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Ordering(com.google.common.collect.Ordering) Function.identity(java.util.function.Function.identity) StageExecution(io.trino.execution.scheduler.StageExecution) ExchangeNode(io.trino.sql.planner.plan.ExchangeNode) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) PlanFragment(io.trino.sql.planner.PlanFragment) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 4 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class LegacyPhasedExecutionSchedule method extractPhases.

@VisibleForTesting
static List<Set<PlanFragmentId>> extractPhases(Collection<PlanFragment> fragments) {
    // Build a graph where the plan fragments are vertexes and the edges represent
    // a before -> after relationship.  For example, a join hash build has an edge
    // to the join probe.
    DirectedGraph<PlanFragmentId, DefaultEdge> graph = new DefaultDirectedGraph<>(DefaultEdge.class);
    fragments.forEach(fragment -> graph.addVertex(fragment.getId()));
    Visitor visitor = new Visitor(fragments, graph);
    for (PlanFragment fragment : fragments) {
        visitor.processFragment(fragment.getId());
    }
    // Computes all the strongly connected components of the directed graph.
    // These are the "phases" which hold the set of fragments that must be started
    // at the same time to avoid deadlock.
    List<Set<PlanFragmentId>> components = new StrongConnectivityInspector<>(graph).stronglyConnectedSets();
    Map<PlanFragmentId, Set<PlanFragmentId>> componentMembership = new HashMap<>();
    for (Set<PlanFragmentId> component : components) {
        for (PlanFragmentId planFragmentId : component) {
            componentMembership.put(planFragmentId, component);
        }
    }
    // build graph of components (phases)
    DirectedGraph<Set<PlanFragmentId>, DefaultEdge> componentGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
    components.forEach(componentGraph::addVertex);
    for (DefaultEdge edge : graph.edgeSet()) {
        PlanFragmentId source = graph.getEdgeSource(edge);
        PlanFragmentId target = graph.getEdgeTarget(edge);
        Set<PlanFragmentId> from = componentMembership.get(source);
        Set<PlanFragmentId> to = componentMembership.get(target);
        if (!from.equals(to)) {
            // the topological order iterator below doesn't include vertices that have self-edges, so don't add them
            componentGraph.addEdge(from, to);
        }
    }
    List<Set<PlanFragmentId>> schedulePhases = ImmutableList.copyOf(new TopologicalOrderIterator<>(componentGraph));
    return schedulePhases;
}
Also used : HashSet(java.util.HashSet) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) PlanVisitor(io.trino.sql.planner.plan.PlanVisitor) DefaultDirectedGraph(org.jgrapht.graph.DefaultDirectedGraph) HashMap(java.util.HashMap) DefaultEdge(org.jgrapht.graph.DefaultEdge) PlanFragment(io.trino.sql.planner.PlanFragment) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 5 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class StageTaskSourceFactory method getExchangeForHandleMap.

private static IdentityHashMap<ExchangeSourceHandle, Exchange> getExchangeForHandleMap(Map<PlanFragmentId, Exchange> sourceExchanges, Multimap<PlanFragmentId, ExchangeSourceHandle> exchangeSourceHandles) {
    IdentityHashMap<ExchangeSourceHandle, Exchange> exchangeForHandle = new IdentityHashMap<>();
    for (Map.Entry<PlanFragmentId, ExchangeSourceHandle> entry : exchangeSourceHandles.entries()) {
        PlanFragmentId fragmentId = entry.getKey();
        ExchangeSourceHandle handle = entry.getValue();
        Exchange exchange = sourceExchanges.get(fragmentId);
        requireNonNull(exchange, "Exchange not found for fragment " + fragmentId);
        exchangeForHandle.put(handle, exchange);
    }
    return exchangeForHandle;
}
Also used : Exchange(io.trino.spi.exchange.Exchange) ExchangeSourceHandle(io.trino.spi.exchange.ExchangeSourceHandle) IdentityHashMap(java.util.IdentityHashMap) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) Map(java.util.Map) IdentityHashMap(java.util.IdentityHashMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Aggregations

PlanFragmentId (io.trino.sql.planner.plan.PlanFragmentId)22 PlanFragment (io.trino.sql.planner.PlanFragment)15 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)10 RemoteSourceNode (io.trino.sql.planner.plan.RemoteSourceNode)10 Test (org.testng.annotations.Test)7 PartitioningScheme (io.trino.sql.planner.PartitioningScheme)6 Symbol (io.trino.sql.planner.Symbol)6 ImmutableMap (com.google.common.collect.ImmutableMap)5 ImmutableSet (com.google.common.collect.ImmutableSet)5 FragmentsEdge (io.trino.execution.scheduler.policy.PhasedExecutionSchedule.FragmentsEdge)5 PlanUtils.createBroadcastAndPartitionedJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastAndPartitionedJoinPlanFragment)5 PlanUtils.createBroadcastJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastJoinPlanFragment)5 PlanUtils.createJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createJoinPlanFragment)5 PlanUtils.createTableScanPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createTableScanPlanFragment)5 JoinNode (io.trino.sql.planner.plan.JoinNode)5 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)4 ExchangeSourceHandle (io.trino.spi.exchange.ExchangeSourceHandle)4 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)3