Search in sources :

Example 1 with SCHEDULED

use of com.facebook.presto.execution.StageExecutionState.SCHEDULED in project presto by prestodb.

the class LegacySqlQueryScheduler method schedule.

private void schedule() {
    if (!scheduling.compareAndSet(false, true)) {
        // still scheduling the previous batch of stages
        return;
    }
    List<StageExecutionAndScheduler> scheduledStageExecutions = new ArrayList<>();
    try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) {
        Set<StageId> completedStages = new HashSet<>();
        List<ExecutionSchedule> sectionExecutionSchedules = new LinkedList<>();
        while (!Thread.currentThread().isInterrupted()) {
            // remove finished section
            sectionExecutionSchedules.removeIf(ExecutionSchedule::isFinished);
            // try to pull more section that are ready to be run
            List<StreamingPlanSection> sectionsReadyForExecution = getSectionsReadyForExecution();
            // all finished
            if (sectionsReadyForExecution.isEmpty() && sectionExecutionSchedules.isEmpty()) {
                break;
            }
            List<List<StageExecutionAndScheduler>> sectionStageExecutions = getStageExecutions(sectionsReadyForExecution);
            sectionStageExecutions.forEach(scheduledStageExecutions::addAll);
            sectionStageExecutions.stream().map(executionInfos -> executionInfos.stream().collect(toImmutableList())).map(executionPolicy::createExecutionSchedule).forEach(sectionExecutionSchedules::add);
            while (sectionExecutionSchedules.stream().noneMatch(ExecutionSchedule::isFinished)) {
                List<ListenableFuture<?>> blockedStages = new ArrayList<>();
                List<StageExecutionAndScheduler> executionsToSchedule = sectionExecutionSchedules.stream().flatMap(schedule -> schedule.getStagesToSchedule().stream()).collect(toImmutableList());
                for (StageExecutionAndScheduler stageExecutionAndScheduler : executionsToSchedule) {
                    SqlStageExecution stageExecution = stageExecutionAndScheduler.getStageExecution();
                    StageId stageId = stageExecution.getStageExecutionId().getStageId();
                    stageExecution.beginScheduling();
                    // perform some scheduling work
                    ScheduleResult result = stageExecutionAndScheduler.getStageScheduler().schedule();
                    // Track leaf tasks if partial results are enabled
                    if (isPartialResultsEnabled(session) && stageExecutionAndScheduler.getStageExecution().getFragment().isLeaf()) {
                        for (RemoteTask task : result.getNewTasks()) {
                            partialResultQueryTaskTracker.trackTask(task);
                            task.addFinalTaskInfoListener(partialResultQueryTaskTracker::recordTaskFinish);
                        }
                    }
                    // modify parent and children based on the results of the scheduling
                    if (result.isFinished()) {
                        stageExecution.schedulingComplete();
                    } else if (!result.getBlocked().isDone()) {
                        blockedStages.add(result.getBlocked());
                    }
                    stageExecutionAndScheduler.getStageLinkage().processScheduleResults(stageExecution.getState(), result.getNewTasks());
                    schedulerStats.getSplitsScheduledPerIteration().add(result.getSplitsScheduled());
                    if (result.getBlockedReason().isPresent()) {
                        switch(result.getBlockedReason().get()) {
                            case WRITER_SCALING:
                                // no-op
                                break;
                            case WAITING_FOR_SOURCE:
                                schedulerStats.getWaitingForSource().update(1);
                                break;
                            case SPLIT_QUEUES_FULL:
                                schedulerStats.getSplitQueuesFull().update(1);
                                break;
                            case MIXED_SPLIT_QUEUES_FULL_AND_WAITING_FOR_SOURCE:
                                schedulerStats.getMixedSplitQueuesFullAndWaitingForSource().update(1);
                                break;
                            case NO_ACTIVE_DRIVER_GROUP:
                                schedulerStats.getNoActiveDriverGroup().update(1);
                                break;
                            default:
                                throw new UnsupportedOperationException("Unknown blocked reason: " + result.getBlockedReason().get());
                        }
                    }
                }
                // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel)
                boolean stageFinishedExecution = false;
                for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) {
                    SqlStageExecution stageExecution = stageExecutionInfo.getStageExecution();
                    StageId stageId = stageExecution.getStageExecutionId().getStageId();
                    if (!completedStages.contains(stageId) && stageExecution.getState().isDone()) {
                        stageExecutionInfo.getStageLinkage().processScheduleResults(stageExecution.getState(), ImmutableSet.of());
                        completedStages.add(stageId);
                        stageFinishedExecution = true;
                    }
                }
                // if any stage has just finished execution try to pull more sections for scheduling
                if (stageFinishedExecution) {
                    break;
                }
                // wait for a state change and then schedule again
                if (!blockedStages.isEmpty()) {
                    try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) {
                        tryGetFutureValue(whenAnyComplete(blockedStages), 1, SECONDS);
                    }
                    for (ListenableFuture<?> blockedStage : blockedStages) {
                        blockedStage.cancel(true);
                    }
                }
            }
        }
        for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) {
            StageExecutionState state = stageExecutionInfo.getStageExecution().getState();
            if (state != SCHEDULED && state != RUNNING && !state.isDone()) {
                throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage execution %s is in state %s", stageExecutionInfo.getStageExecution().getStageExecutionId(), state));
            }
        }
        scheduling.set(false);
        // Inform the tracker that task scheduling has completed
        partialResultQueryTaskTracker.completeTaskScheduling();
        if (!getSectionsReadyForExecution().isEmpty()) {
            startScheduling();
        }
    } catch (Throwable t) {
        scheduling.set(false);
        queryStateMachine.transitionToFailed(t);
        throw t;
    } finally {
        RuntimeException closeError = new RuntimeException();
        for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) {
            try {
                stageExecutionInfo.getStageScheduler().close();
            } catch (Throwable t) {
                queryStateMachine.transitionToFailed(t);
                // Self-suppression not permitted
                if (closeError != t) {
                    closeError.addSuppressed(t);
                }
            }
        }
        if (closeError.getSuppressed().length > 0) {
            throw closeError;
        }
    }
}
Also used : BasicStageExecutionStats(com.facebook.presto.execution.BasicStageExecutionStats) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) WarningCollector(com.facebook.presto.spi.WarningCollector) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) Traverser.forTree(com.google.common.graph.Traverser.forTree) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SchedulingOrderVisitor.scheduleOrder(com.facebook.presto.sql.planner.SchedulingOrderVisitor.scheduleOrder) SystemSessionProperties.getMaxConcurrentMaterializations(com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) SplitSourceFactory(com.facebook.presto.sql.planner.SplitSourceFactory) BROADCAST_PARTITION_ID(com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) StageInfo(com.facebook.presto.execution.StageInfo) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) Map(java.util.Map) MoreFutures.whenAnyComplete(com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete) URI(java.net.URI) PlanPrinter.jsonFragmentPlan(com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonFragmentPlan) PLANNED(com.facebook.presto.execution.StageExecutionState.PLANNED) RemoteTaskFactory(com.facebook.presto.execution.RemoteTaskFactory) ImmutableSet(com.google.common.collect.ImmutableSet) StageExecutionInfo(com.facebook.presto.execution.StageExecutionInfo) RUNNING(com.facebook.presto.execution.StageExecutionState.RUNNING) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) DataSize(io.airlift.units.DataSize) List(java.util.List) MoreFutures.tryGetFutureValue(com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) PlanVariableAllocator(com.facebook.presto.sql.planner.PlanVariableAllocator) FAILED(com.facebook.presto.execution.StageExecutionState.FAILED) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) StageId(com.facebook.presto.execution.StageId) Iterables(com.google.common.collect.Iterables) Logger(com.facebook.airlift.log.Logger) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) SystemSessionProperties.isPartialResultsEnabled(com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SystemSessionProperties.getPartialResultsCompletionRatioThreshold(com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) SCHEDULED(com.facebook.presto.execution.StageExecutionState.SCHEDULED) AtomicReference(java.util.concurrent.atomic.AtomicReference) SystemSessionProperties.isRuntimeOptimizerEnabled(com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) TimeStat(com.facebook.airlift.stats.TimeStat) SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier(com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier) ABORTED(com.facebook.presto.execution.StageExecutionState.ABORTED) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Objects.requireNonNull(java.util.Objects.requireNonNull) BasicStageExecutionStats.aggregateBasicStageStats(com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) LinkedList(java.util.LinkedList) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ExecutorService(java.util.concurrent.ExecutorService) CANCELED(com.facebook.presto.execution.StageExecutionState.CANCELED) SubPlan(com.facebook.presto.sql.planner.SubPlan) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) StreamingPlanSection.extractStreamingSections(com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections) Session(com.facebook.presto.Session) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpUriBuilder.uriBuilderFrom(com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom) Streams.stream(com.google.common.collect.Streams.stream) PlanNode(com.facebook.presto.spi.plan.PlanNode) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskId(com.facebook.presto.execution.TaskId) ROOT_FRAGMENT_ID(com.facebook.presto.sql.planner.PlanFragmenter.ROOT_FRAGMENT_ID) FINISHED(com.facebook.presto.execution.StageExecutionState.FINISHED) QueryState(com.facebook.presto.execution.QueryState) QueryStateMachine(com.facebook.presto.execution.QueryStateMachine) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PartialResultQueryManager(com.facebook.presto.execution.PartialResultQueryManager) Metadata(com.facebook.presto.metadata.Metadata) StageId(com.facebook.presto.execution.StageId) ArrayList(java.util.ArrayList) PrestoException(com.facebook.presto.spi.PrestoException) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) LinkedList(java.util.LinkedList) HashSet(java.util.HashSet) RemoteTask(com.facebook.presto.execution.RemoteTask) TimeStat(com.facebook.airlift.stats.TimeStat) LinkedList(java.util.LinkedList) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Example 2 with SCHEDULED

use of com.facebook.presto.execution.StageExecutionState.SCHEDULED in project presto by prestodb.

the class SqlQueryScheduler method schedule.

private void schedule() {
    if (!scheduling.compareAndSet(false, true)) {
        // still scheduling the previous batch of stages
        return;
    }
    List<StageExecutionAndScheduler> scheduledStageExecutions = new ArrayList<>();
    try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) {
        Set<StageId> completedStages = new HashSet<>();
        List<ExecutionSchedule> executionSchedules = new LinkedList<>();
        while (!Thread.currentThread().isInterrupted()) {
            // remove finished section
            executionSchedules.removeIf(ExecutionSchedule::isFinished);
            // try to pull more section that are ready to be run
            List<StreamingPlanSection> sectionsReadyForExecution = getSectionsReadyForExecution();
            // all finished
            if (sectionsReadyForExecution.isEmpty() && executionSchedules.isEmpty()) {
                break;
            }
            // Apply runtime CBO on the ready sections before creating SectionExecutions.
            List<SectionExecution> sectionExecutions = createStageExecutions(sectionsReadyForExecution.stream().map(this::tryCostBasedOptimize).collect(toImmutableList()));
            if (queryStateMachine.isDone()) {
                sectionExecutions.forEach(SectionExecution::abort);
                break;
            }
            sectionExecutions.forEach(sectionExecution -> scheduledStageExecutions.addAll(sectionExecution.getSectionStages()));
            sectionExecutions.stream().map(SectionExecution::getSectionStages).map(executionPolicy::createExecutionSchedule).forEach(executionSchedules::add);
            while (!executionSchedules.isEmpty() && executionSchedules.stream().noneMatch(ExecutionSchedule::isFinished)) {
                List<ListenableFuture<?>> blockedStages = new ArrayList<>();
                List<StageExecutionAndScheduler> executionsToSchedule = executionSchedules.stream().flatMap(schedule -> schedule.getStagesToSchedule().stream()).collect(toImmutableList());
                for (StageExecutionAndScheduler executionAndScheduler : executionsToSchedule) {
                    executionAndScheduler.getStageExecution().beginScheduling();
                    // perform some scheduling work
                    ScheduleResult result = executionAndScheduler.getStageScheduler().schedule();
                    // Track leaf tasks if partial results are enabled
                    if (isPartialResultsEnabled(session) && executionAndScheduler.getStageExecution().getFragment().isLeaf()) {
                        for (RemoteTask task : result.getNewTasks()) {
                            partialResultQueryTaskTracker.trackTask(task);
                            task.addFinalTaskInfoListener(partialResultQueryTaskTracker::recordTaskFinish);
                        }
                    }
                    // modify parent and children based on the results of the scheduling
                    if (result.isFinished()) {
                        executionAndScheduler.getStageExecution().schedulingComplete();
                    } else if (!result.getBlocked().isDone()) {
                        blockedStages.add(result.getBlocked());
                    }
                    executionAndScheduler.getStageLinkage().processScheduleResults(executionAndScheduler.getStageExecution().getState(), result.getNewTasks());
                    schedulerStats.getSplitsScheduledPerIteration().add(result.getSplitsScheduled());
                    if (result.getBlockedReason().isPresent()) {
                        switch(result.getBlockedReason().get()) {
                            case WRITER_SCALING:
                                // no-op
                                break;
                            case WAITING_FOR_SOURCE:
                                schedulerStats.getWaitingForSource().update(1);
                                break;
                            case SPLIT_QUEUES_FULL:
                                schedulerStats.getSplitQueuesFull().update(1);
                                break;
                            case MIXED_SPLIT_QUEUES_FULL_AND_WAITING_FOR_SOURCE:
                                schedulerStats.getMixedSplitQueuesFullAndWaitingForSource().update(1);
                                break;
                            case NO_ACTIVE_DRIVER_GROUP:
                                schedulerStats.getNoActiveDriverGroup().update(1);
                                break;
                            default:
                                throw new UnsupportedOperationException("Unknown blocked reason: " + result.getBlockedReason().get());
                        }
                    }
                }
                // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel)
                boolean stageFinishedExecution = false;
                for (StageExecutionAndScheduler stageExecutionAndScheduler : scheduledStageExecutions) {
                    SqlStageExecution stageExecution = stageExecutionAndScheduler.getStageExecution();
                    StageId stageId = stageExecution.getStageExecutionId().getStageId();
                    if (!completedStages.contains(stageId) && stageExecution.getState().isDone()) {
                        stageExecutionAndScheduler.getStageLinkage().processScheduleResults(stageExecution.getState(), ImmutableSet.of());
                        completedStages.add(stageId);
                        stageFinishedExecution = true;
                    }
                }
                // if any stage has just finished execution try to pull more sections for scheduling
                if (stageFinishedExecution) {
                    break;
                }
                // wait for a state change and then schedule again
                if (!blockedStages.isEmpty()) {
                    try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) {
                        tryGetFutureValue(whenAnyComplete(blockedStages), 1, SECONDS);
                    }
                    for (ListenableFuture<?> blockedStage : blockedStages) {
                        blockedStage.cancel(true);
                    }
                }
            }
        }
        for (StageExecutionAndScheduler stageExecutionAndScheduler : scheduledStageExecutions) {
            StageExecutionState state = stageExecutionAndScheduler.getStageExecution().getState();
            if (state != SCHEDULED && state != RUNNING && !state.isDone()) {
                throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage execution %s is in state %s", stageExecutionAndScheduler.getStageExecution().getStageExecutionId(), state));
            }
        }
        scheduling.set(false);
        // Inform the tracker that task scheduling has completed
        partialResultQueryTaskTracker.completeTaskScheduling();
        if (!getSectionsReadyForExecution().isEmpty()) {
            startScheduling();
        }
    } catch (Throwable t) {
        scheduling.set(false);
        queryStateMachine.transitionToFailed(t);
        throw t;
    } finally {
        RuntimeException closeError = new RuntimeException();
        for (StageExecutionAndScheduler stageExecutionAndScheduler : scheduledStageExecutions) {
            try {
                stageExecutionAndScheduler.getStageScheduler().close();
            } catch (Throwable t) {
                queryStateMachine.transitionToFailed(t);
                // Self-suppression not permitted
                if (closeError != t) {
                    closeError.addSuppressed(t);
                }
            }
        }
        if (closeError.getSuppressed().length > 0) {
            throw closeError;
        }
    }
}
Also used : BasicStageExecutionStats(com.facebook.presto.execution.BasicStageExecutionStats) WarningCollector(com.facebook.presto.spi.WarningCollector) ListMultimap(com.google.common.collect.ListMultimap) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SchedulingOrderVisitor.scheduleOrder(com.facebook.presto.sql.planner.SchedulingOrderVisitor.scheduleOrder) SystemSessionProperties.getMaxConcurrentMaterializations(com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations) SplitSourceFactory(com.facebook.presto.sql.planner.SplitSourceFactory) StageInfo(com.facebook.presto.execution.StageInfo) InternalNodeManager(com.facebook.presto.metadata.InternalNodeManager) Map(java.util.Map) PlanPrinter.jsonFragmentPlan(com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonFragmentPlan) RemoteTaskFactory(com.facebook.presto.execution.RemoteTaskFactory) RUNNING(com.facebook.presto.execution.StageExecutionState.RUNNING) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) PlanVariableAllocator(com.facebook.presto.sql.planner.PlanVariableAllocator) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) FAILED(com.facebook.presto.execution.StageExecutionState.FAILED) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) SystemSessionProperties.isRuntimeOptimizerEnabled(com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled) ArrayList(java.util.ArrayList) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) CANCELED(com.facebook.presto.execution.StageExecutionState.CANCELED) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) StreamingPlanSection.extractStreamingSections(com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections) Session(com.facebook.presto.Session) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpUriBuilder.uriBuilderFrom(com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom) Streams.stream(com.google.common.collect.Streams.stream) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskId(com.facebook.presto.execution.TaskId) ROOT_FRAGMENT_ID(com.facebook.presto.sql.planner.PlanFragmenter.ROOT_FRAGMENT_ID) FINISHED(com.facebook.presto.execution.StageExecutionState.FINISHED) QueryState(com.facebook.presto.execution.QueryState) Metadata(com.facebook.presto.metadata.Metadata) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) Traverser.forTree(com.google.common.graph.Traverser.forTree) OutputBufferId(com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId) BROADCAST_PARTITION_ID(com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) OutputBuffers.createDiscardingOutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers) Duration(io.airlift.units.Duration) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StageExecutionInfo.unscheduledExecutionInfo(com.facebook.presto.execution.StageExecutionInfo.unscheduledExecutionInfo) MoreFutures.whenAnyComplete(com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete) URI(java.net.URI) ImmutableSet(com.google.common.collect.ImmutableSet) StageExecutionInfo(com.facebook.presto.execution.StageExecutionInfo) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Iterables.getLast(com.google.common.collect.Iterables.getLast) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) Preconditions.checkState(com.google.common.base.Preconditions.checkState) DataSize(io.airlift.units.DataSize) List(java.util.List) MoreFutures.tryGetFutureValue(com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue) Optional(java.util.Optional) StageId(com.facebook.presto.execution.StageId) Logger(com.facebook.airlift.log.Logger) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) SystemSessionProperties.isPartialResultsEnabled(com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) SystemSessionProperties.getPartialResultsCompletionRatioThreshold(com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) ExecutionFailureInfo(com.facebook.presto.execution.ExecutionFailureInfo) PrestoException(com.facebook.presto.spi.PrestoException) SCHEDULED(com.facebook.presto.execution.StageExecutionState.SCHEDULED) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) TimeStat(com.facebook.airlift.stats.TimeStat) SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier(com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) BasicStageExecutionStats.aggregateBasicStageStats(com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats) LinkedList(java.util.LinkedList) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ExecutorService(java.util.concurrent.ExecutorService) VerifyException(com.google.common.base.VerifyException) SubPlan(com.facebook.presto.sql.planner.SubPlan) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) PlanNode(com.facebook.presto.spi.plan.PlanNode) RECOVERABLE_ERROR_CODES(com.facebook.presto.execution.SqlStageExecution.RECOVERABLE_ERROR_CODES) QueryStateMachine(com.facebook.presto.execution.QueryStateMachine) Collections(java.util.Collections) SystemSessionProperties.getMaxStageRetries(com.facebook.presto.SystemSessionProperties.getMaxStageRetries) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PartialResultQueryManager(com.facebook.presto.execution.PartialResultQueryManager) StageId(com.facebook.presto.execution.StageId) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) PrestoException(com.facebook.presto.spi.PrestoException) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) HashSet(java.util.HashSet) RemoteTask(com.facebook.presto.execution.RemoteTask) TimeStat(com.facebook.airlift.stats.TimeStat) LinkedList(java.util.LinkedList) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) StageExecutionState(com.facebook.presto.execution.StageExecutionState) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Aggregations

MoreFutures.tryGetFutureValue (com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue)2 MoreFutures.whenAnyComplete (com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete)2 SetThreadName (com.facebook.airlift.concurrent.SetThreadName)2 HttpUriBuilder.uriBuilderFrom (com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom)2 Logger (com.facebook.airlift.log.Logger)2 TimeStat (com.facebook.airlift.stats.TimeStat)2 Session (com.facebook.presto.Session)2 SystemSessionProperties.getMaxConcurrentMaterializations (com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations)2 SystemSessionProperties.getPartialResultsCompletionRatioThreshold (com.facebook.presto.SystemSessionProperties.getPartialResultsCompletionRatioThreshold)2 SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier (com.facebook.presto.SystemSessionProperties.getPartialResultsMaxExecutionTimeMultiplier)2 SystemSessionProperties.isPartialResultsEnabled (com.facebook.presto.SystemSessionProperties.isPartialResultsEnabled)2 SystemSessionProperties.isRuntimeOptimizerEnabled (com.facebook.presto.SystemSessionProperties.isRuntimeOptimizerEnabled)2 BasicStageExecutionStats (com.facebook.presto.execution.BasicStageExecutionStats)2 BasicStageExecutionStats.aggregateBasicStageStats (com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats)2 LocationFactory (com.facebook.presto.execution.LocationFactory)2 PartialResultQueryManager (com.facebook.presto.execution.PartialResultQueryManager)2 QueryState (com.facebook.presto.execution.QueryState)2 QueryStateMachine (com.facebook.presto.execution.QueryStateMachine)2 RemoteTask (com.facebook.presto.execution.RemoteTask)2 RemoteTaskFactory (com.facebook.presto.execution.RemoteTaskFactory)2