Search in sources :

Example 1 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class LegacySqlQueryScheduler method updateStageExecutions.

/**
 * Utility function that rebuild a StreamingPlanSection, re-create stageExecutionAndScheduler for each of its stage, and finally update the stageExecutions map.
 */
private void updateStageExecutions(StreamingPlanSection section, Map<PlanFragment, PlanFragment> oldToNewFragment) {
    StreamingPlanSection newSection = new StreamingPlanSection(rewriteStreamingSubPlan(section.getPlan(), oldToNewFragment), section.getChildren());
    PlanFragment sectionRootFragment = newSection.getPlan().getFragment();
    Optional<int[]> bucketToPartition;
    OutputBuffers outputBuffers;
    ExchangeLocationsConsumer locationsConsumer;
    if (isRootFragment(sectionRootFragment)) {
        bucketToPartition = Optional.of(new int[1]);
        outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
        OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
    } else {
        bucketToPartition = Optional.empty();
        outputBuffers = createDiscardingOutputBuffers();
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
        };
    }
    SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, newSection, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, 0);
    addStateChangeListeners(sectionExecution);
    Map<StageId, StageExecutionAndScheduler> updatedStageExecutions = sectionExecution.getSectionStages().stream().collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity()));
    synchronized (this) {
        stageExecutions.putAll(updatedStageExecutions);
    }
}
Also used : BasicStageExecutionStats(com.facebook.presto.execution.BasicStageExecutionStats) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) WarningCollector(com.facebook.presto.spi.WarningCollector) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) Traverser.forTree(com.google.common.graph.Traverser.forTree) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SchedulingOrderVisitor.scheduleOrder(com.facebook.presto.sql.planner.SchedulingOrderVisitor.scheduleOrder) SystemSessionProperties.getMaxConcurrentMaterializations(com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) SplitSourceFactory(com.facebook.presto.sql.planner.SplitSourceFactory) BROADCAST_PARTITION_ID(com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) StageInfo(com.facebook.presto.execution.StageInfo) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) Map(java.util.Map) MoreFutures.whenAnyComplete(com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete) URI(java.net.URI) PlanPrinter.jsonFragmentPlan(com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonFragmentPlan) PLANNED(com.facebook.presto.execution.StageExecutionState.PLANNED) RemoteTaskFactory(com.facebook.presto.execution.RemoteTaskFactory) ImmutableSet(com.google.common.collect.ImmutableSet) StageExecutionInfo(com.facebook.presto.execution.StageExecutionInfo) RUNNING(com.facebook.presto.execution.StageExecutionState.RUNNING) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) DataSize(io.airlift.units.DataSize) List(java.util.List) MoreFutures.tryGetFutureValue(com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) PlanVariableAllocator(com.facebook.presto.sql.planner.PlanVariableAllocator) FAILED(com.facebook.presto.execution.StageExecutionState.FAILED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) StageId(com.facebook.presto.execution.StageId) Iterables(com.google.common.collect.Iterables) Logger(com.facebook.airlift.log.Logger) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) SystemSessionProperties.isPartialResultsEnabled(com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SystemSessionProperties.getPartialResultsCompletionRatioThreshold(com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) SCHEDULED(com.facebook.presto.execution.StageExecutionState.SCHEDULED) AtomicReference(java.util.concurrent.atomic.AtomicReference) SystemSessionProperties.isRuntimeOptimizerEnabled(com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TimeStat(com.facebook.airlift.stats.TimeStat) SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier(com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier) ABORTED(com.facebook.presto.execution.StageExecutionState.ABORTED) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Objects.requireNonNull(java.util.Objects.requireNonNull) BasicStageExecutionStats.aggregateBasicStageStats(com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) LinkedList(java.util.LinkedList) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ExecutorService(java.util.concurrent.ExecutorService) CANCELED(com.facebook.presto.execution.StageExecutionState.CANCELED) SubPlan(com.facebook.presto.sql.planner.SubPlan) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) StreamingPlanSection.extractStreamingSections(com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections) Session(com.facebook.presto.Session) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpUriBuilder.uriBuilderFrom(com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom) Streams.stream(com.google.common.collect.Streams.stream) PlanNode(com.facebook.presto.spi.plan.PlanNode) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskId(com.facebook.presto.execution.TaskId) ROOT_FRAGMENT_ID(com.facebook.presto.sql.planner.PlanFragmenter.ROOT_FRAGMENT_ID) FINISHED(com.facebook.presto.execution.StageExecutionState.FINISHED) QueryState(com.facebook.presto.execution.QueryState) QueryStateMachine(com.facebook.presto.execution.QueryStateMachine) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PartialResultQueryManager(com.facebook.presto.execution.PartialResultQueryManager) Metadata(com.facebook.presto.metadata.Metadata) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) StageId(com.facebook.presto.execution.StageId) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) PlanFragment(com.facebook.presto.sql.planner.PlanFragment)

Example 2 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class ScaledOutputBufferManager method addOutputBuffers.

@SuppressWarnings("ObjectEquality")
@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // so ignore the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, newBuffer.getId());
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}
Also used : OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId)

Example 3 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class SqlQueryExecution method planDistribution.

private void planDistribution(PlanRoot plan) {
    CloseableSplitSourceProvider splitSourceProvider = new CloseableSplitSourceProvider(splitManager::getSplits);
    // ensure split sources are closed
    stateMachine.addStateChangeListener(state -> {
        if (state.isDone()) {
            splitSourceProvider.close();
        }
    });
    // if query was canceled, skip creating scheduler
    if (stateMachine.isDone()) {
        return;
    }
    SubPlan outputStagePlan = plan.getRoot();
    // record output field
    stateMachine.setColumns(((OutputNode) outputStagePlan.getFragment().getRoot()).getColumnNames(), outputStagePlan.getFragment().getTypes());
    PartitioningHandle partitioningHandle = outputStagePlan.getFragment().getPartitioningScheme().getPartitioning().getHandle();
    OutputBuffers rootOutputBuffers;
    if (isSpoolingOutputBufferEnabled(getSession())) {
        rootOutputBuffers = createSpoolingOutputBuffers();
    } else {
        rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle).withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    }
    SplitSourceFactory splitSourceFactory = new SplitSourceFactory(splitSourceProvider, stateMachine.getWarningCollector());
    // build the stage execution objects (this doesn't schedule execution)
    SqlQuerySchedulerInterface scheduler = isUseLegacyScheduler(getSession()) ? LegacySqlQueryScheduler.createSqlQueryScheduler(locationFactory, executionPolicy, queryExecutor, schedulerStats, sectionExecutionFactory, remoteTaskFactory, splitSourceFactory, stateMachine.getSession(), metadata.getFunctionAndTypeManager(), stateMachine, outputStagePlan, rootOutputBuffers, plan.isSummarizeTaskInfos(), runtimePlanOptimizers, stateMachine.getWarningCollector(), idAllocator, variableAllocator.get(), planChecker, metadata, sqlParser, partialResultQueryManager) : SqlQueryScheduler.createSqlQueryScheduler(locationFactory, executionPolicy, queryExecutor, schedulerStats, sectionExecutionFactory, remoteTaskFactory, splitSourceFactory, internalNodeManager, stateMachine.getSession(), stateMachine, outputStagePlan, plan.isSummarizeTaskInfos(), metadata.getFunctionAndTypeManager(), runtimePlanOptimizers, stateMachine.getWarningCollector(), idAllocator, variableAllocator.get(), planChecker, metadata, sqlParser, partialResultQueryManager);
    queryScheduler.set(scheduler);
    // directly since the callback may have already fired
    if (stateMachine.isDone()) {
        scheduler.abort();
        queryScheduler.set(null);
    }
}
Also used : OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers.createSpoolingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createSpoolingOutputBuffers) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) CloseableSplitSourceProvider(com.facebook.presto.split.CloseableSplitSourceProvider) SplitSourceFactory(com.facebook.presto.sql.planner.SplitSourceFactory) SqlQuerySchedulerInterface(com.facebook.presto.execution.scheduler.SqlQuerySchedulerInterface) PartitioningHandle(com.facebook.presto.sql.planner.PartitioningHandle) SubPlan(com.facebook.presto.sql.planner.SubPlan)

Example 4 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class SqlStageExecution method scheduleTask.

private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, Multimap<PlanNodeId, Split> sourceSplits) {
    checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId);
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    initialSplits.putAll(sourceSplits);
    sourceTasks.forEach((planNodeId, task) -> {
        TaskStatus status = task.getTaskStatus();
        if (status.getState() != TaskState.FINISHED) {
            initialSplits.put(planNodeId, createRemoteSplitFor(taskId, task.getRemoteTaskLocation(), task.getTaskId()));
        }
    });
    OutputBuffers outputBuffers = this.outputBuffers.get();
    checkState(outputBuffers != null, "Initial output buffers must be set before a task can be scheduled");
    RemoteTask task = remoteTaskFactory.createRemoteTask(session, taskId, node, planFragment, initialSplits.build(), outputBuffers, nodeTaskMap.createTaskStatsTracker(node, taskId), summarizeTaskInfo, tableWriteInfo);
    completeSources.forEach(task::noMoreSplits);
    allTasks.add(taskId);
    tasks.computeIfAbsent(node, key -> newConcurrentHashSet()).add(task);
    nodeTaskMap.addTask(node, task);
    task.addStateChangeListener(new StageTaskListener(taskId));
    task.addFinalTaskInfoListener(this::updateFinalTaskInfo);
    if (!stateMachine.getState().isDone()) {
        task.start();
    } else {
        // stage finished while we were scheduling this task
        task.abort();
    }
    return task;
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) RemoteSourceNode(com.facebook.presto.sql.planner.plan.RemoteSourceNode) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) Duration(io.airlift.units.Duration) TableWriteInfo(com.facebook.presto.execution.scheduler.TableWriteInfo) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) HashMultimap(com.google.common.collect.HashMultimap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) URI(java.net.URI) ImmutableSet(com.google.common.collect.ImmutableSet) REMOTE_TASK_MISMATCH(com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_MISMATCH) SplitSchedulerStats(com.facebook.presto.execution.scheduler.SplitSchedulerStats) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ThreadSafe(javax.annotation.concurrent.ThreadSafe) PAGE_TRANSPORT_TIMEOUT(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT) GuardedBy(javax.annotation.concurrent.GuardedBy) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) DataSize(io.airlift.units.DataSize) List(java.util.List) TOO_MANY_REQUESTS_FAILED(com.facebook.presto.spi.StandardErrorCode.TOO_MANY_REQUESTS_FAILED) Entry(java.util.Map.Entry) Optional(java.util.Optional) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) ErrorCode(com.facebook.presto.spi.ErrorCode) NANOSECONDS(java.util.concurrent.TimeUnit.NANOSECONDS) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SystemSessionProperties.getMaxFailedTaskPercentage(com.facebook.presto.SystemSessionProperties.getMaxFailedTaskPercentage) PrestoException(com.facebook.presto.spi.PrestoException) Multimap(com.google.common.collect.Multimap) AtomicReference(java.util.concurrent.atomic.AtomicReference) REMOTE_HOST_GONE(com.facebook.presto.spi.StandardErrorCode.REMOTE_HOST_GONE) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Objects.requireNonNull(java.util.Objects.requireNonNull) RemoteSplit(com.facebook.presto.split.RemoteSplit) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) GENERIC_RECOVERY_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_RECOVERY_ERROR) RemoteTransactionHandle(com.facebook.presto.metadata.RemoteTransactionHandle) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ExecutorService(java.util.concurrent.ExecutorService) PAGE_TRANSPORT_ERROR(com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_ERROR) Executor(java.util.concurrent.Executor) Session(com.facebook.presto.Session) Sets.newConcurrentHashSet(com.google.common.collect.Sets.newConcurrentHashSet) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) InternalNode(com.facebook.presto.metadata.InternalNode) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) StateChangeListener(com.facebook.presto.execution.StateMachine.StateChangeListener) REMOTE_TASK_ERROR(com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_ERROR) FailureDetector(com.facebook.presto.failureDetector.FailureDetector) REMOTE_CONNECTOR_ID(com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID) Split(com.facebook.presto.metadata.Split) GONE(com.facebook.presto.failureDetector.FailureDetector.State.GONE) BYTE(io.airlift.units.DataSize.Unit.BYTE) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) RemoteSplit(com.facebook.presto.split.RemoteSplit) Split(com.facebook.presto.metadata.Split)

Example 5 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class BroadcastOutputBufferManager method addOutputBuffers.

@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}
Also used : OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId)

Aggregations

OutputBuffers (com.facebook.presto.execution.buffer.OutputBuffers)11 OutputBuffers.createInitialEmptyOutputBuffers (com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers)9 OutputBufferId (com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId)5 DataSize (io.airlift.units.DataSize)5 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)4 Session (com.facebook.presto.Session)3 StageId (com.facebook.presto.execution.StageId)2 PrestoException (com.facebook.presto.spi.PrestoException)2 GENERIC_INTERNAL_ERROR (com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR)2 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)2 PlanFragment (com.facebook.presto.sql.planner.PlanFragment)2 SplitSourceFactory (com.facebook.presto.sql.planner.SplitSourceFactory)2 SubPlan (com.facebook.presto.sql.planner.SubPlan)2 PlanFragmentId (com.facebook.presto.sql.planner.plan.PlanFragmentId)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Iterables.getOnlyElement (com.google.common.collect.Iterables.getOnlyElement)2 Duration (io.airlift.units.Duration)2 String.format (java.lang.String.format)2