Search in sources :

Example 1 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, List<PlanFragment> allFragments, Metadata metadata) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream().flatMap(f -> f.getSymbols().entrySet().stream()).distinct().collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, valuePrinter, fragment.getStatsAndCosts(), planNodeStats, metadata).toText(verbose, 1)).append("\n");
    return builder.toString();
}
Also used : TableDeleteNode(io.prestosql.sql.planner.plan.TableDeleteNode) SortNode(io.prestosql.sql.planner.plan.SortNode) SubPlan(io.prestosql.sql.planner.SubPlan) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) REUSE_STRATEGY_PRODUCER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER) TypeProvider(io.prestosql.sql.planner.TypeProvider) NullableValue(io.prestosql.spi.predicate.NullableValue) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) CTEScanNode(io.prestosql.spi.plan.CTEScanNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) TableUpdateNode(io.prestosql.sql.planner.plan.TableUpdateNode) Map(java.util.Map) OutputNode(io.prestosql.sql.planner.plan.OutputNode) Partitioning(io.prestosql.sql.planner.Partitioning) TopNRankingNumberNode(io.prestosql.sql.planner.plan.TopNRankingNumberNode) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TypedSymbol(io.prestosql.sql.planner.planprinter.NodeRepresentation.TypedSymbol) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) CreateIndexNode(io.prestosql.sql.planner.plan.CreateIndexNode) SortExpressionContext(io.prestosql.sql.planner.SortExpressionContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) DynamicFilters(io.prestosql.sql.DynamicFilters) IndexSourceNode(io.prestosql.sql.planner.plan.IndexSourceNode) PlanNode(io.prestosql.spi.plan.PlanNode) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Collectors.joining(java.util.stream.Collectors.joining) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Domain(io.prestosql.spi.predicate.Domain) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) Joiner(com.google.common.base.Joiner) GroupIdNode(io.prestosql.spi.plan.GroupIdNode) Iterables(com.google.common.collect.Iterables) IntersectNode(io.prestosql.spi.plan.IntersectNode) Marker(io.prestosql.spi.predicate.Marker) DynamicFilters.extractDynamicFilters(io.prestosql.sql.DynamicFilters.extractDynamicFilters) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) AssignUniqueId(io.prestosql.sql.planner.plan.AssignUniqueId) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Session(io.prestosql.Session) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Assignments(io.prestosql.spi.plan.Assignments) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TextRenderer.formatPositions(io.prestosql.sql.planner.planprinter.TextRenderer.formatPositions) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SampleNode(io.prestosql.sql.planner.plan.SampleNode) WindowNode(io.prestosql.spi.plan.WindowNode) LimitNode(io.prestosql.spi.plan.LimitNode) Expression(io.prestosql.sql.tree.Expression) OffsetNode(io.prestosql.sql.planner.plan.OffsetNode) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) Scope(io.prestosql.sql.planner.plan.ExchangeNode.Scope) UpdateIndexNode(io.prestosql.sql.planner.plan.UpdateIndexNode) Duration(io.airlift.units.Duration) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) PlanCostEstimate(io.prestosql.cost.PlanCostEstimate) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) FilterNode(io.prestosql.spi.plan.FilterNode) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString) TextRenderer.formatDouble(io.prestosql.sql.planner.planprinter.TextRenderer.formatDouble) Type(io.prestosql.spi.type.Type) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableSet(com.google.common.collect.ImmutableSet) JoinNodeUtils(io.prestosql.sql.planner.optimizations.JoinNodeUtils) Collection(java.util.Collection) Streams(com.google.common.collect.Streams) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) RowNumberNode(io.prestosql.sql.planner.plan.RowNumberNode) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) TopNNode(io.prestosql.spi.plan.TopNNode) StageInfo.getAllStages(io.prestosql.execution.StageInfo.getAllStages) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) ExceptNode(io.prestosql.spi.plan.ExceptNode) Arrays.stream(java.util.Arrays.stream) StageStats(io.prestosql.execution.StageStats) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) StatsAndCosts(io.prestosql.cost.StatsAndCosts) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TRUE_LITERAL(io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ImmutableList(com.google.common.collect.ImmutableList) OrderingScheme(io.prestosql.spi.plan.OrderingScheme) GraphvizPrinter(io.prestosql.util.GraphvizPrinter) Verify.verify(com.google.common.base.Verify.verify) Range(io.prestosql.spi.predicate.Range) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) MarkDistinctNode(io.prestosql.spi.plan.MarkDistinctNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) REUSE_STRATEGY_CONSUMER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER) TableInfo(io.prestosql.execution.TableInfo) CaseFormat(com.google.common.base.CaseFormat) GroupReference(io.prestosql.spi.plan.GroupReference) TupleDomain(io.prestosql.spi.predicate.TupleDomain) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Aggregation(io.prestosql.spi.plan.AggregationNode.Aggregation) RowExpression(io.prestosql.spi.relation.RowExpression) SortExpressionExtractor(io.prestosql.sql.planner.SortExpressionExtractor) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) Entry(java.util.Map.Entry) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) StageStats(io.prestosql.execution.StageStats) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeProvider(io.prestosql.sql.planner.TypeProvider) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString)

Example 2 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(StageInfo outputStageInfo, ValuePrinter valuePrinter, boolean verbose, Metadata metadata) {
    Map<PlanNodeId, TableInfo> tableInfos = getAllStages(Optional.of(outputStageInfo)).stream().map(StageInfo::getTables).map(Map::entrySet).flatMap(Collection::stream).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = getAllStages(Optional.of(outputStageInfo));
    List<PlanFragment> allFragments = allStages.stream().map(StageInfo::getPlan).collect(toImmutableList());
    Map<PlanNodeId, PlanNodeStats> aggregatedStats = aggregateStageStats(allStages);
    for (StageInfo stageInfo : allStages) {
        builder.append(formatFragment(tableScanNode -> tableInfos.get(tableScanNode.getId()), valuePrinter, stageInfo.getPlan(), Optional.of(stageInfo), Optional.of(aggregatedStats), verbose, allFragments, metadata));
    }
    return builder.toString();
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableDeleteNode(io.prestosql.sql.planner.plan.TableDeleteNode) SortNode(io.prestosql.sql.planner.plan.SortNode) SubPlan(io.prestosql.sql.planner.SubPlan) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) REUSE_STRATEGY_PRODUCER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER) TypeProvider(io.prestosql.sql.planner.TypeProvider) NullableValue(io.prestosql.spi.predicate.NullableValue) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) CTEScanNode(io.prestosql.spi.plan.CTEScanNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) TableUpdateNode(io.prestosql.sql.planner.plan.TableUpdateNode) Map(java.util.Map) OutputNode(io.prestosql.sql.planner.plan.OutputNode) Partitioning(io.prestosql.sql.planner.Partitioning) TopNRankingNumberNode(io.prestosql.sql.planner.plan.TopNRankingNumberNode) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TypedSymbol(io.prestosql.sql.planner.planprinter.NodeRepresentation.TypedSymbol) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) CreateIndexNode(io.prestosql.sql.planner.plan.CreateIndexNode) SortExpressionContext(io.prestosql.sql.planner.SortExpressionContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) DynamicFilters(io.prestosql.sql.DynamicFilters) IndexSourceNode(io.prestosql.sql.planner.plan.IndexSourceNode) PlanNode(io.prestosql.spi.plan.PlanNode) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Collectors.joining(java.util.stream.Collectors.joining) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Domain(io.prestosql.spi.predicate.Domain) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) Joiner(com.google.common.base.Joiner) GroupIdNode(io.prestosql.spi.plan.GroupIdNode) Iterables(com.google.common.collect.Iterables) IntersectNode(io.prestosql.spi.plan.IntersectNode) Marker(io.prestosql.spi.predicate.Marker) DynamicFilters.extractDynamicFilters(io.prestosql.sql.DynamicFilters.extractDynamicFilters) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) AssignUniqueId(io.prestosql.sql.planner.plan.AssignUniqueId) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Session(io.prestosql.Session) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Assignments(io.prestosql.spi.plan.Assignments) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TextRenderer.formatPositions(io.prestosql.sql.planner.planprinter.TextRenderer.formatPositions) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SampleNode(io.prestosql.sql.planner.plan.SampleNode) WindowNode(io.prestosql.spi.plan.WindowNode) LimitNode(io.prestosql.spi.plan.LimitNode) Expression(io.prestosql.sql.tree.Expression) OffsetNode(io.prestosql.sql.planner.plan.OffsetNode) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) Scope(io.prestosql.sql.planner.plan.ExchangeNode.Scope) UpdateIndexNode(io.prestosql.sql.planner.plan.UpdateIndexNode) Duration(io.airlift.units.Duration) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) PlanCostEstimate(io.prestosql.cost.PlanCostEstimate) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) FilterNode(io.prestosql.spi.plan.FilterNode) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString) TextRenderer.formatDouble(io.prestosql.sql.planner.planprinter.TextRenderer.formatDouble) Type(io.prestosql.spi.type.Type) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableSet(com.google.common.collect.ImmutableSet) JoinNodeUtils(io.prestosql.sql.planner.optimizations.JoinNodeUtils) Collection(java.util.Collection) Streams(com.google.common.collect.Streams) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) RowNumberNode(io.prestosql.sql.planner.plan.RowNumberNode) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) TopNNode(io.prestosql.spi.plan.TopNNode) StageInfo.getAllStages(io.prestosql.execution.StageInfo.getAllStages) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) ExceptNode(io.prestosql.spi.plan.ExceptNode) Arrays.stream(java.util.Arrays.stream) StageStats(io.prestosql.execution.StageStats) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) StatsAndCosts(io.prestosql.cost.StatsAndCosts) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TRUE_LITERAL(io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ImmutableList(com.google.common.collect.ImmutableList) OrderingScheme(io.prestosql.spi.plan.OrderingScheme) GraphvizPrinter(io.prestosql.util.GraphvizPrinter) Verify.verify(com.google.common.base.Verify.verify) Range(io.prestosql.spi.predicate.Range) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) MarkDistinctNode(io.prestosql.spi.plan.MarkDistinctNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) REUSE_STRATEGY_CONSUMER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER) TableInfo(io.prestosql.execution.TableInfo) CaseFormat(com.google.common.base.CaseFormat) GroupReference(io.prestosql.spi.plan.GroupReference) TupleDomain(io.prestosql.spi.predicate.TupleDomain) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Aggregation(io.prestosql.spi.plan.AggregationNode.Aggregation) RowExpression(io.prestosql.spi.relation.RowExpression) SortExpressionExtractor(io.prestosql.sql.planner.SortExpressionExtractor) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) StageInfo(io.prestosql.execution.StageInfo) TableInfo(io.prestosql.execution.TableInfo) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) PlanFragment(io.prestosql.sql.planner.PlanFragment)

Example 3 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class Query method globalUniqueNodes.

public static Set<String> globalUniqueNodes(StageInfo stageInfo, boolean getSpilledNodes) {
    boolean isSpilledNode = false;
    if (stageInfo == null) {
        return ImmutableSet.of();
    }
    ImmutableSet.Builder<String> nodes = ImmutableSet.builder();
    for (TaskInfo task : stageInfo.getTasks()) {
        if (getSpilledNodes) {
            for (PipelineStats pipeline : task.getStats().getPipelines()) {
                isSpilledNode = pipeline.getOperatorSummaries().stream().filter(operatorStats -> operatorStats.getSpilledDataSize().toBytes() > 0).collect(Collectors.toList()).size() > 0;
                if (isSpilledNode) {
                    break;
                }
            }
        }
        // todo add nodeId to TaskInfo
        if (isSpilledNode || !getSpilledNodes) {
            URI uri = task.getTaskStatus().getSelf();
            nodes.add(uri.getHost() + ":" + uri.getPort());
        }
    }
    for (StageInfo subStage : stageInfo.getSubStages()) {
        nodes.addAll(globalUniqueNodes(subStage, getSpilledNodes));
    }
    return nodes.build();
}
Also used : TaskInfo(io.prestosql.execution.TaskInfo) PipelineStats(io.prestosql.operator.PipelineStats) ImmutableSet(com.google.common.collect.ImmutableSet) StageInfo(io.prestosql.execution.StageInfo) URI(java.net.URI)

Example 4 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class Query method toStatementStats.

private StatementStats toStatementStats(QueryInfo queryInfo) {
    QueryStats queryStats = queryInfo.getQueryStats();
    // Dont print any more stats for Async Query,
    StageInfo outputStage = queryInfo.isRunningAsync() ? null : queryInfo.getOutputStage().orElse(null);
    return StatementStats.builder().setState(queryInfo.getState().toString()).setQueued(queryInfo.getState() == QueryState.QUEUED).setScheduled(queryInfo.isScheduled()).setNodes(globalUniqueNodes(outputStage, false).size()).setTotalSplits(queryStats.getTotalDrivers()).setQueuedSplits(queryStats.getQueuedDrivers()).setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()).setCompletedSplits(queryStats.getCompletedDrivers()).setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()).setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()).setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()).setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()).setProcessedRows(queryStats.getRawInputPositions()).setProcessedBytes(queryStats.getRawInputDataSize().toBytes()).setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()).setSpilledBytes(queryStats.getSpilledDataSize().toBytes()).setSpilledReadTimeMillis(queryStats.getSpilledReadTime().toMillis()).setSpilledWriteTimeMillis(queryStats.getSpilledWriteTime().toMillis()).setSpilledNodes(globalUniqueNodes(outputStage, true).size()).setRootStage(toStageStats(outputStage)).setSnapshotStats(toSnapshotStats(queryInfo.getQueryId())).build();
}
Also used : QueryStats(io.prestosql.execution.QueryStats) StageInfo(io.prestosql.execution.StageInfo)

Example 5 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class ExplainAnalyzeOperator method getSubStages.

private static void getSubStages(StageId stageId, StageInfo rootStage, ImmutableList.Builder<StageInfo> collector, boolean add) {
    boolean status = add;
    if (rootStage.getStageId().equals(stageId)) {
        status = true;
    }
    List<StageInfo> subStages = rootStage.getSubStages();
    for (StageInfo subStage : subStages) {
        getSubStages(stageId, subStage, collector, status);
    }
    if (status && !rootStage.getStageId().equals(stageId)) {
        collector.add(rootStage);
    }
}
Also used : StageInfo(io.prestosql.execution.StageInfo)

Aggregations

StageInfo (io.prestosql.execution.StageInfo)9 ImmutableList (com.google.common.collect.ImmutableList)4 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)4 ImmutableSet (com.google.common.collect.ImmutableSet)4 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 Preconditions.checkState (com.google.common.base.Preconditions.checkState)3 Verify.verify (com.google.common.base.Verify.verify)3 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)3 Iterables (com.google.common.collect.Iterables)3 Duration (io.airlift.units.Duration)3 Session (io.prestosql.Session)3 CaseFormat (com.google.common.base.CaseFormat)2 UPPER_UNDERSCORE (com.google.common.base.CaseFormat.UPPER_UNDERSCORE)2 Functions (com.google.common.base.Functions)2 Joiner (com.google.common.base.Joiner)2 Lists (com.google.common.collect.Lists)2 Streams (com.google.common.collect.Streams)2 PlanCostEstimate (io.prestosql.cost.PlanCostEstimate)2 PlanNodeStatsEstimate (io.prestosql.cost.PlanNodeStatsEstimate)2 StatsAndCosts (io.prestosql.cost.StatsAndCosts)2