Search in sources :

Example 1 with PipelineStats

use of com.facebook.presto.operator.PipelineStats in project presto by prestodb.

the class PlanNodeStatsSummarizer method getPlanNodeStats.

private static List<PlanNodeStats> getPlanNodeStats(TaskStats taskStats) {
    // Best effort to reconstruct the plan nodes from operators.
    // Because stats are collected separately from query execution,
    // it's possible that some or all of them are missing or out of date.
    // For example, a LIMIT clause can cause a query to finish before stats
    // are collected from the leaf stages.
    Set<PlanNodeId> planNodeIds = new HashSet<>();
    Map<PlanNodeId, Long> planNodeInputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeInputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeRawInputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeRawInputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeScheduledMillis = new HashMap<>();
    Map<PlanNodeId, Long> planNodeCpuMillis = new HashMap<>();
    Map<PlanNodeId, Map<String, OperatorInputStats>> operatorInputStats = new HashMap<>();
    Map<PlanNodeId, Map<String, OperatorHashCollisionsStats>> operatorHashCollisionsStats = new HashMap<>();
    Map<PlanNodeId, WindowOperatorStats> windowNodeStats = new HashMap<>();
    for (PipelineStats pipelineStats : taskStats.getPipelines()) {
        // Due to eventual consistently collected stats, these could be empty
        if (pipelineStats.getOperatorSummaries().isEmpty()) {
            continue;
        }
        Set<PlanNodeId> processedNodes = new HashSet<>();
        PlanNodeId inputPlanNode = pipelineStats.getOperatorSummaries().iterator().next().getPlanNodeId();
        PlanNodeId outputPlanNode = getLast(pipelineStats.getOperatorSummaries()).getPlanNodeId();
        // Gather input statistics
        for (OperatorStats operatorStats : pipelineStats.getOperatorSummaries()) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            planNodeIds.add(planNodeId);
            long scheduledMillis = operatorStats.getAddInputWall().toMillis() + operatorStats.getGetOutputWall().toMillis() + operatorStats.getFinishWall().toMillis();
            planNodeScheduledMillis.merge(planNodeId, scheduledMillis, Long::sum);
            long cpuMillis = operatorStats.getAddInputCpu().toMillis() + operatorStats.getGetOutputCpu().toMillis() + operatorStats.getFinishCpu().toMillis();
            planNodeCpuMillis.merge(planNodeId, cpuMillis, Long::sum);
            // A pipeline like hash build before join might link to another "internal" pipelines which provide actual input for this plan node
            if (operatorStats.getPlanNodeId().equals(inputPlanNode) && !pipelineStats.isInputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            operatorInputStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new OperatorInputStats(operatorStats.getTotalDrivers(), operatorStats.getInputPositions(), operatorStats.getSumSquaredInputPositions())), (map1, map2) -> mergeMaps(map1, map2, OperatorInputStats::merge));
            if (operatorStats.getInfo() instanceof HashCollisionsInfo) {
                HashCollisionsInfo hashCollisionsInfo = (HashCollisionsInfo) operatorStats.getInfo();
                operatorHashCollisionsStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new OperatorHashCollisionsStats(hashCollisionsInfo.getWeightedHashCollisions(), hashCollisionsInfo.getWeightedSumSquaredHashCollisions(), hashCollisionsInfo.getWeightedExpectedHashCollisions())), (map1, map2) -> mergeMaps(map1, map2, OperatorHashCollisionsStats::merge));
            }
            // The only statistics we have for Window Functions are very low level, thus displayed only in VERBOSE mode
            if (operatorStats.getInfo() instanceof WindowInfo) {
                WindowInfo windowInfo = (WindowInfo) operatorStats.getInfo();
                windowNodeStats.merge(planNodeId, WindowOperatorStats.create(windowInfo), (left, right) -> left.mergeWith(right));
            }
            planNodeInputPositions.merge(planNodeId, operatorStats.getInputPositions(), Long::sum);
            planNodeInputBytes.merge(planNodeId, operatorStats.getInputDataSize().toBytes(), Long::sum);
            planNodeRawInputPositions.merge(planNodeId, operatorStats.getRawInputPositions(), Long::sum);
            planNodeRawInputBytes.merge(planNodeId, operatorStats.getRawInputDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
        // Gather output statistics
        processedNodes.clear();
        for (OperatorStats operatorStats : reverse(pipelineStats.getOperatorSummaries())) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            // An "internal" pipeline like a hash build, links to another pipeline which is the actual output for this plan node
            if (operatorStats.getPlanNodeId().equals(outputPlanNode) && !pipelineStats.isOutputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            planNodeOutputPositions.merge(planNodeId, operatorStats.getOutputPositions(), Long::sum);
            planNodeOutputBytes.merge(planNodeId, operatorStats.getOutputDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
    }
    List<PlanNodeStats> stats = new ArrayList<>();
    for (PlanNodeId planNodeId : planNodeIds) {
        if (!planNodeInputPositions.containsKey(planNodeId)) {
            continue;
        }
        PlanNodeStats nodeStats;
        // It's possible there will be no output stats because all the pipelines that we observed were non-output.
        // For example in a query like SELECT * FROM a JOIN b ON c = d LIMIT 1
        // It's possible to observe stats after the build starts, but before the probe does
        // and therefore only have scheduled time, but no output stats
        long outputPositions = planNodeOutputPositions.getOrDefault(planNodeId, 0L);
        if (operatorHashCollisionsStats.containsKey(planNodeId)) {
            nodeStats = new HashCollisionPlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), operatorHashCollisionsStats.get(planNodeId));
        } else if (windowNodeStats.containsKey(planNodeId)) {
            nodeStats = new WindowPlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), windowNodeStats.get(planNodeId));
        } else {
            nodeStats = new PlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), planNodeRawInputPositions.get(planNodeId), succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId));
        }
        stats.add(nodeStats);
    }
    return stats;
}
Also used : PipelineStats(com.facebook.presto.operator.PipelineStats) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Duration(io.airlift.units.Duration) OperatorStats(com.facebook.presto.operator.OperatorStats) HashCollisionsInfo(com.facebook.presto.operator.HashCollisionsInfo) WindowInfo(com.facebook.presto.operator.WindowInfo) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Example 2 with PipelineStats

use of com.facebook.presto.operator.PipelineStats in project presto by prestodb.

the class PlanPrinter method getPlanNodeStats.

private static List<PlanNodeStats> getPlanNodeStats(TaskStats taskStats) {
    // Best effort to reconstruct the plan nodes from operators.
    // Because stats are collected separately from query execution,
    // it's possible that some or all of them are missing or out of date.
    // For example, a LIMIT clause can cause a query to finish before stats
    // are collected from the leaf stages.
    Map<PlanNodeId, Long> planNodeInputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeInputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeWallMillis = new HashMap<>();
    Map<PlanNodeId, Map<String, OperatorInputStats>> operatorInputStats = new HashMap<>();
    Map<PlanNodeId, Map<String, OperatorHashCollisionsStats>> operatorHashCollisionsStats = new HashMap<>();
    for (PipelineStats pipelineStats : taskStats.getPipelines()) {
        // Due to eventual consistently collected stats, these could be empty
        if (pipelineStats.getOperatorSummaries().isEmpty()) {
            continue;
        }
        Set<PlanNodeId> processedNodes = new HashSet<>();
        PlanNodeId inputPlanNode = pipelineStats.getOperatorSummaries().iterator().next().getPlanNodeId();
        PlanNodeId outputPlanNode = getLast(pipelineStats.getOperatorSummaries()).getPlanNodeId();
        // Gather input statistics
        for (OperatorStats operatorStats : pipelineStats.getOperatorSummaries()) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            long wall = operatorStats.getAddInputWall().toMillis() + operatorStats.getGetOutputWall().toMillis() + operatorStats.getFinishWall().toMillis();
            planNodeWallMillis.merge(planNodeId, wall, Long::sum);
            // A pipeline like hash build before join might link to another "internal" pipelines which provide actual input for this plan node
            if (operatorStats.getPlanNodeId().equals(inputPlanNode) && !pipelineStats.isInputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            operatorInputStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new OperatorInputStats(operatorStats.getTotalDrivers(), operatorStats.getInputPositions(), operatorStats.getSumSquaredInputPositions())), PlanPrinter::mergeOperatorInputStatsMaps);
            if (operatorStats.getInfo() instanceof HashCollisionsInfo) {
                HashCollisionsInfo hashCollisionsInfo = (HashCollisionsInfo) operatorStats.getInfo();
                operatorHashCollisionsStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new OperatorHashCollisionsStats(hashCollisionsInfo.getWeightedHashCollisions(), hashCollisionsInfo.getWeightedSumSquaredHashCollisions(), hashCollisionsInfo.getWeightedExpectedHashCollisions())), PlanPrinter::mergeOperatorHashCollisionsStatsMaps);
            }
            planNodeInputPositions.merge(planNodeId, operatorStats.getInputPositions(), Long::sum);
            planNodeInputBytes.merge(planNodeId, operatorStats.getInputDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
        // Gather output statistics
        processedNodes.clear();
        for (OperatorStats operatorStats : reverse(pipelineStats.getOperatorSummaries())) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            // An "internal" pipeline like a hash build, links to another pipeline which is the actual output for this plan node
            if (operatorStats.getPlanNodeId().equals(outputPlanNode) && !pipelineStats.isOutputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            planNodeOutputPositions.merge(planNodeId, operatorStats.getOutputPositions(), Long::sum);
            planNodeOutputBytes.merge(planNodeId, operatorStats.getOutputDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
    }
    List<PlanNodeStats> stats = new ArrayList<>();
    for (Map.Entry<PlanNodeId, Long> entry : planNodeWallMillis.entrySet()) {
        PlanNodeId planNodeId = entry.getKey();
        stats.add(new PlanNodeStats(planNodeId, new Duration(planNodeWallMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), // and therefore only have wall time, but no output stats
        planNodeOutputPositions.getOrDefault(planNodeId, 0L), succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), // Only some operators emit hash collisions statistics
        operatorHashCollisionsStats.getOrDefault(planNodeId, emptyMap())));
    }
    return stats;
}
Also used : PipelineStats(com.facebook.presto.operator.PipelineStats) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Duration(io.airlift.units.Duration) OperatorStats(com.facebook.presto.operator.OperatorStats) HashCollisionsInfo(com.facebook.presto.operator.HashCollisionsInfo) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) Map(java.util.Map) Collectors.toMap(java.util.stream.Collectors.toMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap) HashSet(java.util.HashSet)

Example 3 with PipelineStats

use of com.facebook.presto.operator.PipelineStats in project presto by prestodb.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier, Supplier<Iterable<StageInfo>> subStageInfosSupplier) {
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    List<StageInfo> subStageInfos = ImmutableList.copyOf(subStageInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeMemory = 0;
    long totalMemoryReservation = 0;
    long peakMemoryReservation = getPeakMemoryInBytes();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalUserTime = 0;
    long totalBlockedTime = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeMemory += taskStats.getCumulativeMemory();
        totalMemoryReservation += taskStats.getMemoryReservation().toBytes();
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalUserTime += taskStats.getTotalUserTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), scheduleTaskDistribution.snapshot(), addSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, cumulativeMemory, succinctBytes(totalMemoryReservation), succinctBytes(peakMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalUserTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, subStageInfos, failureInfo);
}
Also used : PipelineStats(com.facebook.presto.operator.PipelineStats) BlockedReason(com.facebook.presto.operator.BlockedReason) HashMap(java.util.HashMap) OperatorStats(com.facebook.presto.operator.OperatorStats) TaskStats(com.facebook.presto.operator.TaskStats) HashSet(java.util.HashSet)

Example 4 with PipelineStats

use of com.facebook.presto.operator.PipelineStats in project presto by prestodb.

the class StageExecutionInfo method create.

public static StageExecutionInfo create(StageExecutionId stageExecutionId, StageExecutionState state, Optional<ExecutionFailureInfo> failureInfo, List<TaskInfo> taskInfos, DateTime schedulingComplete, DistributionSnapshot getSplitDistribution, DataSize peakUserMemoryReservation, DataSize peakNodeTotalMemoryReservation, int finishedLifespans, int totalLifespans) {
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    double cumulativeUserMemory = 0;
    double cumulativeTotalMemory = 0;
    long userMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long retriedCpuTime = 0;
    long totalBlockedTime = 0;
    long totalAllocation = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long bufferedDataSize = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    int fullGcCount = 0;
    int fullGcTaskCount = 0;
    int minFullGcSec = 0;
    int maxFullGcSec = 0;
    int totalFullGcSec = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    RuntimeStats mergedRuntimeStats = new RuntimeStats();
    mergedRuntimeStats.addMetricValueIgnoreZero(GET_SPLITS_TIME_NANOS, (long) getSplitDistribution.getTotal());
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        blockedDrivers += taskStats.getBlockedDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        cumulativeTotalMemory += taskStats.getCumulativeTotalMemory();
        long taskUserMemory = taskStats.getUserMemoryReservationInBytes();
        long taskSystemMemory = taskStats.getSystemMemoryReservationInBytes();
        userMemoryReservation += taskUserMemory;
        totalMemoryReservation += taskUserMemory + taskSystemMemory;
        totalScheduledTime += taskStats.getTotalScheduledTimeInNanos();
        totalCpuTime += taskStats.getTotalCpuTimeInNanos();
        if (state == FINISHED && taskInfo.getTaskStatus().getState() == TaskState.FAILED) {
            retriedCpuTime += taskStats.getTotalCpuTimeInNanos();
        }
        totalBlockedTime += taskStats.getTotalBlockedTimeInNanos();
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        totalAllocation += taskStats.getTotalAllocationInBytes();
        rawInputDataSize += taskStats.getRawInputDataSizeInBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSizeInBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
        outputDataSize += taskStats.getOutputDataSizeInBytes();
        outputPositions += taskStats.getOutputPositions();
        physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSizeInBytes();
        fullGcCount += taskStats.getFullGcCount();
        fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
        int gcSec = toIntExact(MILLISECONDS.toSeconds(taskStats.getFullGcTimeInMillis()));
        totalFullGcSec += gcSec;
        minFullGcSec = min(minFullGcSec, gcSec);
        maxFullGcSec = max(maxFullGcSec, gcSec);
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
        mergedRuntimeStats.mergeWith(taskStats.getRuntimeStats());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.DRIVER_COUNT_PER_TASK, taskStats.getTotalDrivers());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_ELAPSED_TIME_NANOS, taskStats.getElapsedTimeInNanos());
        mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_QUEUED_TIME_NANOS, taskStats.getQueuedTimeInNanos());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_SCHEDULED_TIME_NANOS, taskStats.getTotalScheduledTimeInNanos());
        mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_BLOCKED_TIME_NANOS, taskStats.getTotalBlockedTimeInNanos());
    }
    StageExecutionStats stageExecutionStats = new StageExecutionStats(schedulingComplete, getSplitDistribution, totalTasks, runningTasks, completedTasks, totalLifespans, finishedLifespans, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakNodeTotalMemoryReservation, succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(retriedCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), new StageGcStatistics(stageExecutionId.getStageId().getId(), stageExecutionId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()), mergedRuntimeStats);
    return new StageExecutionInfo(state, stageExecutionStats, taskInfos, failureInfo);
}
Also used : PipelineStats(com.facebook.presto.operator.PipelineStats) BlockedReason(com.facebook.presto.operator.BlockedReason) HashMap(java.util.HashMap) RuntimeStats(com.facebook.presto.common.RuntimeStats) OperatorStats(com.facebook.presto.operator.OperatorStats) TaskStats(com.facebook.presto.operator.TaskStats) StageGcStatistics(com.facebook.presto.spi.eventlistener.StageGcStatistics) HashSet(java.util.HashSet)

Aggregations

OperatorStats (com.facebook.presto.operator.OperatorStats)4 PipelineStats (com.facebook.presto.operator.PipelineStats)4 HashMap (java.util.HashMap)4 HashSet (java.util.HashSet)4 BlockedReason (com.facebook.presto.operator.BlockedReason)2 HashCollisionsInfo (com.facebook.presto.operator.HashCollisionsInfo)2 TaskStats (com.facebook.presto.operator.TaskStats)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 Duration (io.airlift.units.Duration)2 ArrayList (java.util.ArrayList)2 Map (java.util.Map)2 RuntimeStats (com.facebook.presto.common.RuntimeStats)1 WindowInfo (com.facebook.presto.operator.WindowInfo)1 StageGcStatistics (com.facebook.presto.spi.eventlistener.StageGcStatistics)1 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)1 PlanNodeId (com.facebook.presto.sql.planner.plan.PlanNodeId)1 Collections.emptyMap (java.util.Collections.emptyMap)1 Collectors.toMap (java.util.stream.Collectors.toMap)1