Search in sources :

Example 1 with StageExecutionStats

use of com.facebook.presto.execution.StageExecutionStats in project presto by prestodb.

the class PlanPrinter method formatFragment.

private static String formatFragment(FunctionAndTypeManager functionAndTypeManager, Session session, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageExecutionStats stageExecutionStats = stageInfo.get().getLatestAttemptExecutionInfo().getStats();
        List<TaskInfo> tasks = stageInfo.get().getLatestAttemptExecutionInfo().getTasks();
        double avgPositionsPerTask = tasks.stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = tasks.stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / tasks.size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageExecutionStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageExecutionStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageExecutionStats.getProcessedInputPositions()), stageExecutionStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageExecutionStats.getOutputPositions()), stageExecutionStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    builder.append(indentString(1));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(partitioningScheme.getPartitioning().getArguments()), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(partitioningScheme.getPartitioning().getArguments()), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    TypeProvider typeProvider = TypeProvider.fromVariables(fragment.getVariables());
    builder.append(textLogicalPlan(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), functionAndTypeManager, fragment.getStatsAndCosts(), session, planNodeStats, 1, verbose)).append("\n");
    return builder.toString();
}
Also used : TaskInfo(com.facebook.presto.execution.TaskInfo) ValuesNode(com.facebook.presto.spi.plan.ValuesNode) StageInfo(com.facebook.presto.execution.StageInfo) StageExecutionDescriptor(com.facebook.presto.operator.StageExecutionDescriptor) Map(java.util.Map) IndexJoinNode(com.facebook.presto.sql.planner.plan.IndexJoinNode) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) OrderingScheme(com.facebook.presto.spi.plan.OrderingScheme) StageExecutionStats(com.facebook.presto.execution.StageExecutionStats) StageInfo.getAllStages(com.facebook.presto.execution.StageInfo.getAllStages) InterpretedFunctionInvoker(com.facebook.presto.sql.InterpretedFunctionInvoker) LateralJoinNode(com.facebook.presto.sql.planner.plan.LateralJoinNode) RowNumberNode(com.facebook.presto.sql.planner.plan.RowNumberNode) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TextRenderer.formatDouble(com.facebook.presto.sql.planner.planPrinter.TextRenderer.formatDouble) Set(java.util.Set) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ConnectorSession(com.facebook.presto.spi.ConnectorSession) LimitNode(com.facebook.presto.spi.plan.LimitNode) Stream(java.util.stream.Stream) OperatorNotFoundException(com.facebook.presto.metadata.OperatorNotFoundException) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) TextRenderer.indentString(com.facebook.presto.sql.planner.planPrinter.TextRenderer.indentString) ApplyNode(com.facebook.presto.sql.planner.plan.ApplyNode) AssignUniqueId(com.facebook.presto.sql.planner.plan.AssignUniqueId) CAST(com.facebook.presto.metadata.CastType.CAST) Joiner(com.google.common.base.Joiner) Iterables(com.google.common.collect.Iterables) SINGLE_DISTRIBUTION(com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) Slice(io.airlift.slice.Slice) GroupIdNode(com.facebook.presto.sql.planner.plan.GroupIdNode) Assignments(com.facebook.presto.spi.plan.Assignments) DistinctLimitNode(com.facebook.presto.spi.plan.DistinctLimitNode) PlanCostEstimate(com.facebook.presto.cost.PlanCostEstimate) ArrayList(java.util.ArrayList) TableWriterMergeNode(com.facebook.presto.sql.planner.plan.TableWriterMergeNode) Lists(com.google.common.collect.Lists) LogicalRowExpressions(com.facebook.presto.expressions.LogicalRowExpressions) InternalPlanVisitor(com.facebook.presto.sql.planner.plan.InternalPlanVisitor) TableHandle(com.facebook.presto.spi.TableHandle) FunctionResolution(com.facebook.presto.sql.relational.FunctionResolution) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(com.facebook.presto.cost.PlanNodeStatsEstimate) WindowNode(com.facebook.presto.sql.planner.plan.WindowNode) Session(com.facebook.presto.Session) Domain(com.facebook.presto.common.predicate.Domain) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) SemiJoinNode(com.facebook.presto.sql.planner.plan.SemiJoinNode) TopNNode(com.facebook.presto.spi.plan.TopNNode) PlanNodeStatsSummarizer.aggregateStageStats(com.facebook.presto.sql.planner.planPrinter.PlanNodeStatsSummarizer.aggregateStageStats) TaskInfo(com.facebook.presto.execution.TaskInfo) DynamicFilters.extractDynamicFilters(com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters) ExpressionTreeUtils.createSymbolReference(com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) IntersectNode(com.facebook.presto.spi.plan.IntersectNode) AggregationNode(com.facebook.presto.spi.plan.AggregationNode) ExplainAnalyzeNode(com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode) SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode) RemoteSourceNode(com.facebook.presto.sql.planner.plan.RemoteSourceNode) SortNode(com.facebook.presto.sql.planner.plan.SortNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) MetadataDeleteNode(com.facebook.presto.sql.planner.plan.MetadataDeleteNode) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode) Duration(io.airlift.units.Duration) GraphvizPrinter(com.facebook.presto.util.GraphvizPrinter) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) TopNRowNumberNode(com.facebook.presto.sql.planner.plan.TopNRowNumberNode) IndexSourceNode(com.facebook.presto.sql.planner.plan.IndexSourceNode) CallExpression(com.facebook.presto.spi.relation.CallExpression) OutputNode(com.facebook.presto.sql.planner.plan.OutputNode) ImmutableSet(com.google.common.collect.ImmutableSet) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) Range(com.facebook.presto.common.predicate.Range) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) SampleNode(com.facebook.presto.sql.planner.plan.SampleNode) Optional(java.util.Optional) JoinNodeUtils(com.facebook.presto.sql.planner.optimizations.JoinNodeUtils) Arrays.stream(java.util.Arrays.stream) MarkDistinctNode(com.facebook.presto.spi.plan.MarkDistinctNode) DynamicFilterExtractResult(com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) UnionNode(com.facebook.presto.spi.plan.UnionNode) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) TextRenderer.formatPositions(com.facebook.presto.sql.planner.planPrinter.TextRenderer.formatPositions) RowExpressionDeterminismEvaluator(com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator) Function(java.util.function.Function) ExceptNode(com.facebook.presto.spi.plan.ExceptNode) FilterNode(com.facebook.presto.spi.plan.FilterNode) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) UnnestNode(com.facebook.presto.sql.planner.plan.UnnestNode) Type(com.facebook.presto.common.type.Type) GroupReference(com.facebook.presto.sql.planner.iterative.GroupReference) SourceLocation(com.facebook.presto.spi.SourceLocation) Partitioning(com.facebook.presto.sql.planner.Partitioning) RowExpression(com.facebook.presto.spi.relation.RowExpression) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) SubPlan(com.facebook.presto.sql.planner.SubPlan) CaseFormat(com.google.common.base.CaseFormat) ComparisonExpression(com.facebook.presto.sql.tree.ComparisonExpression) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) DeleteNode(com.facebook.presto.sql.planner.plan.DeleteNode) JsonPlanFragment(com.facebook.presto.sql.planner.planPrinter.JsonRenderer.JsonPlanFragment) PlanNode(com.facebook.presto.spi.plan.PlanNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Expression(com.facebook.presto.sql.tree.Expression) StatsAndCosts(com.facebook.presto.cost.StatsAndCosts) FunctionHandle(com.facebook.presto.spi.function.FunctionHandle) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) EnforceSingleRowNode(com.facebook.presto.sql.planner.plan.EnforceSingleRowNode) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) StatisticsWriterNode(com.facebook.presto.sql.planner.plan.StatisticsWriterNode) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) StageExecutionStats(com.facebook.presto.execution.StageExecutionStats)

Example 2 with StageExecutionStats

use of com.facebook.presto.execution.StageExecutionStats in project presto by prestodb.

the class QueryResourceUtil method toStageStats.

private static StageStats toStageStats(StageInfo stageInfo) {
    if (stageInfo == null) {
        return null;
    }
    StageExecutionInfo currentStageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo();
    StageExecutionStats stageExecutionStats = currentStageExecutionInfo.getStats();
    ImmutableList.Builder<StageStats> subStages = ImmutableList.builder();
    for (StageInfo subStage : stageInfo.getSubStages()) {
        subStages.add(toStageStats(subStage));
    }
    Set<String> uniqueNodes = new HashSet<>();
    for (TaskInfo task : currentStageExecutionInfo.getTasks()) {
        // todo add nodeId to TaskInfo
        URI uri = task.getTaskStatus().getSelf();
        uniqueNodes.add(uri.getHost() + ":" + uri.getPort());
    }
    return StageStats.builder().setStageId(String.valueOf(stageInfo.getStageId().getId())).setState(currentStageExecutionInfo.getState().toString()).setDone(currentStageExecutionInfo.getState().isDone()).setNodes(uniqueNodes.size()).setTotalSplits(stageExecutionStats.getTotalDrivers()).setQueuedSplits(stageExecutionStats.getQueuedDrivers()).setRunningSplits(stageExecutionStats.getRunningDrivers() + stageExecutionStats.getBlockedDrivers()).setCompletedSplits(stageExecutionStats.getCompletedDrivers()).setCpuTimeMillis(stageExecutionStats.getTotalCpuTime().toMillis()).setWallTimeMillis(stageExecutionStats.getTotalScheduledTime().toMillis()).setProcessedRows(stageExecutionStats.getRawInputPositions()).setProcessedBytes(stageExecutionStats.getRawInputDataSize().toBytes()).setSubStages(subStages.build()).build();
}
Also used : TaskInfo(com.facebook.presto.execution.TaskInfo) ImmutableList(com.google.common.collect.ImmutableList) StageStats(com.facebook.presto.client.StageStats) StageInfo(com.facebook.presto.execution.StageInfo) StageExecutionInfo(com.facebook.presto.execution.StageExecutionInfo) StageExecutionStats(com.facebook.presto.execution.StageExecutionStats) URI(java.net.URI) HashSet(java.util.HashSet)

Aggregations

StageExecutionStats (com.facebook.presto.execution.StageExecutionStats)2 StageInfo (com.facebook.presto.execution.StageInfo)2 TaskInfo (com.facebook.presto.execution.TaskInfo)2 Session (com.facebook.presto.Session)1 StageStats (com.facebook.presto.client.StageStats)1 Domain (com.facebook.presto.common.predicate.Domain)1 Range (com.facebook.presto.common.predicate.Range)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 Type (com.facebook.presto.common.type.Type)1 VARCHAR (com.facebook.presto.common.type.VarcharType.VARCHAR)1 PlanCostEstimate (com.facebook.presto.cost.PlanCostEstimate)1 PlanNodeStatsEstimate (com.facebook.presto.cost.PlanNodeStatsEstimate)1 StatsAndCosts (com.facebook.presto.cost.StatsAndCosts)1 StageExecutionInfo (com.facebook.presto.execution.StageExecutionInfo)1 StageInfo.getAllStages (com.facebook.presto.execution.StageInfo.getAllStages)1 DynamicFilterExtractResult (com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult)1 DynamicFilters.extractDynamicFilters (com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters)1 LogicalRowExpressions (com.facebook.presto.expressions.LogicalRowExpressions)1 CAST (com.facebook.presto.metadata.CastType.CAST)1 FunctionAndTypeManager (com.facebook.presto.metadata.FunctionAndTypeManager)1