Search in sources :

Example 1 with BROADCAST_PARTITION_ID

use of com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID in project presto by prestodb.

the class LegacySqlQueryScheduler method updateStageExecutions.

/**
 * Utility function that rebuild a StreamingPlanSection, re-create stageExecutionAndScheduler for each of its stage, and finally update the stageExecutions map.
 */
private void updateStageExecutions(StreamingPlanSection section, Map<PlanFragment, PlanFragment> oldToNewFragment) {
    StreamingPlanSection newSection = new StreamingPlanSection(rewriteStreamingSubPlan(section.getPlan(), oldToNewFragment), section.getChildren());
    PlanFragment sectionRootFragment = newSection.getPlan().getFragment();
    Optional<int[]> bucketToPartition;
    OutputBuffers outputBuffers;
    ExchangeLocationsConsumer locationsConsumer;
    if (isRootFragment(sectionRootFragment)) {
        bucketToPartition = Optional.of(new int[1]);
        outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
        OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
    } else {
        bucketToPartition = Optional.empty();
        outputBuffers = createDiscardingOutputBuffers();
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
        };
    }
    SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, newSection, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, 0);
    addStateChangeListeners(sectionExecution);
    Map<StageId, StageExecutionAndScheduler> updatedStageExecutions = sectionExecution.getSectionStages().stream().collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity()));
    synchronized (this) {
        stageExecutions.putAll(updatedStageExecutions);
    }
}
Also used : BasicStageExecutionStats(com.facebook.presto.execution.BasicStageExecutionStats) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) WarningCollector(com.facebook.presto.spi.WarningCollector) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) Traverser.forTree(com.google.common.graph.Traverser.forTree) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SchedulingOrderVisitor.scheduleOrder(com.facebook.presto.sql.planner.SchedulingOrderVisitor.scheduleOrder) SystemSessionProperties.getMaxConcurrentMaterializations(com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) SplitSourceFactory(com.facebook.presto.sql.planner.SplitSourceFactory) BROADCAST_PARTITION_ID(com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) StageInfo(com.facebook.presto.execution.StageInfo) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) Map(java.util.Map) MoreFutures.whenAnyComplete(com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete) URI(java.net.URI) PlanPrinter.jsonFragmentPlan(com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonFragmentPlan) PLANNED(com.facebook.presto.execution.StageExecutionState.PLANNED) RemoteTaskFactory(com.facebook.presto.execution.RemoteTaskFactory) ImmutableSet(com.google.common.collect.ImmutableSet) StageExecutionInfo(com.facebook.presto.execution.StageExecutionInfo) RUNNING(com.facebook.presto.execution.StageExecutionState.RUNNING) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) DataSize(io.airlift.units.DataSize) List(java.util.List) MoreFutures.tryGetFutureValue(com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) PlanVariableAllocator(com.facebook.presto.sql.planner.PlanVariableAllocator) FAILED(com.facebook.presto.execution.StageExecutionState.FAILED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) StageId(com.facebook.presto.execution.StageId) Iterables(com.google.common.collect.Iterables) Logger(com.facebook.airlift.log.Logger) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) SystemSessionProperties.isPartialResultsEnabled(com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SystemSessionProperties.getPartialResultsCompletionRatioThreshold(com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) SCHEDULED(com.facebook.presto.execution.StageExecutionState.SCHEDULED) AtomicReference(java.util.concurrent.atomic.AtomicReference) SystemSessionProperties.isRuntimeOptimizerEnabled(com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TimeStat(com.facebook.airlift.stats.TimeStat) SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier(com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier) ABORTED(com.facebook.presto.execution.StageExecutionState.ABORTED) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Objects.requireNonNull(java.util.Objects.requireNonNull) BasicStageExecutionStats.aggregateBasicStageStats(com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) LinkedList(java.util.LinkedList) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ExecutorService(java.util.concurrent.ExecutorService) CANCELED(com.facebook.presto.execution.StageExecutionState.CANCELED) SubPlan(com.facebook.presto.sql.planner.SubPlan) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) StreamingPlanSection.extractStreamingSections(com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections) Session(com.facebook.presto.Session) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpUriBuilder.uriBuilderFrom(com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom) Streams.stream(com.google.common.collect.Streams.stream) PlanNode(com.facebook.presto.spi.plan.PlanNode) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskId(com.facebook.presto.execution.TaskId) ROOT_FRAGMENT_ID(com.facebook.presto.sql.planner.PlanFragmenter.ROOT_FRAGMENT_ID) FINISHED(com.facebook.presto.execution.StageExecutionState.FINISHED) QueryState(com.facebook.presto.execution.QueryState) QueryStateMachine(com.facebook.presto.execution.QueryStateMachine) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PartialResultQueryManager(com.facebook.presto.execution.PartialResultQueryManager) Metadata(com.facebook.presto.metadata.Metadata) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) StageId(com.facebook.presto.execution.StageId) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) PlanFragment(com.facebook.presto.sql.planner.PlanFragment)

Aggregations

MoreFutures.tryGetFutureValue (com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue)1 MoreFutures.whenAnyComplete (com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete)1 SetThreadName (com.facebook.airlift.concurrent.SetThreadName)1 HttpUriBuilder.uriBuilderFrom (com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom)1 Logger (com.facebook.airlift.log.Logger)1 TimeStat (com.facebook.airlift.stats.TimeStat)1 Session (com.facebook.presto.Session)1 SystemSessionProperties.getMaxConcurrentMaterializations (com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations)1 SystemSessionProperties.getPartialResultsCompletionRatioThreshold (com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold)1 SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier (com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier)1 SystemSessionProperties.isPartialResultsEnabled (com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled)1 SystemSessionProperties.isRuntimeOptimizerEnabled (com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled)1 BasicStageExecutionStats (com.facebook.presto.execution.BasicStageExecutionStats)1 BasicStageExecutionStats.aggregateBasicStageStats (com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats)1 LocationFactory (com.facebook.presto.execution.LocationFactory)1 PartialResultQueryManager (com.facebook.presto.execution.PartialResultQueryManager)1 QueryState (com.facebook.presto.execution.QueryState)1 QueryStateMachine (com.facebook.presto.execution.QueryStateMachine)1 RemoteTask (com.facebook.presto.execution.RemoteTask)1 RemoteTaskFactory (com.facebook.presto.execution.RemoteTaskFactory)1