Examples with OperatorStats com.facebook.presto.operator.OperatorStats used on opensource projects

Example 6 with OperatorStats

use of com.facebook.presto.operator.OperatorStats in project presto by prestodb.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier, Supplier<Iterable<StageInfo>> subStageInfosSupplier) {
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    List<StageInfo> subStageInfos = ImmutableList.copyOf(subStageInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeMemory = 0;
    long totalMemoryReservation = 0;
    long peakMemoryReservation = getPeakMemoryInBytes();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalUserTime = 0;
    long totalBlockedTime = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeMemory += taskStats.getCumulativeMemory();
        totalMemoryReservation += taskStats.getMemoryReservation().toBytes();
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalUserTime += taskStats.getTotalUserTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), scheduleTaskDistribution.snapshot(), addSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, cumulativeMemory, succinctBytes(totalMemoryReservation), succinctBytes(peakMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalUserTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, subStageInfos, failureInfo);
}

Example 7 with OperatorStats

use of com.facebook.presto.operator.OperatorStats in project presto by prestodb.

the class StageExecutionInfo method create.

public static StageExecutionInfo create(StageExecutionId stageExecutionId, StageExecutionState state, Optional<ExecutionFailureInfo> failureInfo, List<TaskInfo> taskInfos, DateTime schedulingComplete, DistributionSnapshot getSplitDistribution, DataSize peakUserMemoryReservation, DataSize peakNodeTotalMemoryReservation, int finishedLifespans, int totalLifespans) {
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    double cumulativeUserMemory = 0;
    double cumulativeTotalMemory = 0;
    long userMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long retriedCpuTime = 0;
    long totalBlockedTime = 0;
    long totalAllocation = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long bufferedDataSize = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    int fullGcCount = 0;
    int fullGcTaskCount = 0;
    int minFullGcSec = 0;
    int maxFullGcSec = 0;
    int totalFullGcSec = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    RuntimeStats mergedRuntimeStats = new RuntimeStats();
    mergedRuntimeStats.addMetricValueIgnoreZero(GET_SPLITS_TIME_NANOS, (long) getSplitDistribution.getTotal());
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        blockedDrivers += taskStats.getBlockedDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        cumulativeTotalMemory += taskStats.getCumulativeTotalMemory();
        long taskUserMemory = taskStats.getUserMemoryReservationInBytes();
        long taskSystemMemory = taskStats.getSystemMemoryReservationInBytes();
        userMemoryReservation += taskUserMemory;
        totalMemoryReservation += taskUserMemory + taskSystemMemory;
        totalScheduledTime += taskStats.getTotalScheduledTimeInNanos();
        totalCpuTime += taskStats.getTotalCpuTimeInNanos();
        if (state == FINISHED && taskInfo.getTaskStatus().getState() == TaskState.FAILED) {
            retriedCpuTime += taskStats.getTotalCpuTimeInNanos();
        }
        totalBlockedTime += taskStats.getTotalBlockedTimeInNanos();
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        totalAllocation += taskStats.getTotalAllocationInBytes();
        rawInputDataSize += taskStats.getRawInputDataSizeInBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSizeInBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
        outputDataSize += taskStats.getOutputDataSizeInBytes();
        outputPositions += taskStats.getOutputPositions();
        physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSizeInBytes();
        fullGcCount += taskStats.getFullGcCount();
        fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
        int gcSec = toIntExact(MILLISECONDS.toSeconds(taskStats.getFullGcTimeInMillis()));
        totalFullGcSec += gcSec;
        minFullGcSec = min(minFullGcSec, gcSec);
        maxFullGcSec = max(maxFullGcSec, gcSec);
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
        mergedRuntimeStats.mergeWith(taskStats.getRuntimeStats());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.DRIVER_COUNT_PER_TASK, taskStats.getTotalDrivers());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_ELAPSED_TIME_NANOS, taskStats.getElapsedTimeInNanos());
        mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_QUEUED_TIME_NANOS, taskStats.getQueuedTimeInNanos());
        mergedRuntimeStats.addMetricValue(RuntimeMetricName.TASK_SCHEDULED_TIME_NANOS, taskStats.getTotalScheduledTimeInNanos());
        mergedRuntimeStats.addMetricValueIgnoreZero(RuntimeMetricName.TASK_BLOCKED_TIME_NANOS, taskStats.getTotalBlockedTimeInNanos());
    }
    StageExecutionStats stageExecutionStats = new StageExecutionStats(schedulingComplete, getSplitDistribution, totalTasks, runningTasks, completedTasks, totalLifespans, finishedLifespans, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakNodeTotalMemoryReservation, succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(retriedCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), new StageGcStatistics(stageExecutionId.getStageId().getId(), stageExecutionId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()), mergedRuntimeStats);
    return new StageExecutionInfo(state, stageExecutionStats, taskInfos, failureInfo);
}

Example 8 with OperatorStats

use of com.facebook.presto.operator.OperatorStats in project presto by prestodb.

the class QueryStats method create.

public static QueryStats create(QueryStateTimer queryStateTimer, Optional<StageInfo> rootStage, int peakRunningTasks, DataSize peakUserMemoryReservation, DataSize peakTotalMemoryReservation, DataSize peakTaskUserMemory, DataSize peakTaskTotalMemory, DataSize peakNodeTotalMemory, RuntimeStats runtimeStats) {
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    double cumulativeUserMemory = 0;
    double cumulativeTotalMemory = 0;
    long userMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long retriedCpuTime = 0;
    long totalBlockedTime = 0;
    long totalAllocation = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long writtenOutputPositions = 0;
    long writtenOutputLogicalDataSize = 0;
    long writtenOutputPhysicalDataSize = 0;
    long writtenIntermediatePhysicalDataSize = 0;
    ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    RuntimeStats mergedRuntimeStats = RuntimeStats.copyOf(runtimeStats);
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageExecutionStats stageExecutionStats = stageInfo.getLatestAttemptExecutionInfo().getStats();
        totalTasks += stageExecutionStats.getTotalTasks();
        runningTasks += stageExecutionStats.getRunningTasks();
        completedTasks += stageExecutionStats.getCompletedTasks();
        totalDrivers += stageExecutionStats.getTotalDrivers();
        queuedDrivers += stageExecutionStats.getQueuedDrivers();
        runningDrivers += stageExecutionStats.getRunningDrivers();
        blockedDrivers += stageExecutionStats.getBlockedDrivers();
        completedDrivers += stageExecutionStats.getCompletedDrivers();
        cumulativeUserMemory += stageExecutionStats.getCumulativeUserMemory();
        cumulativeTotalMemory += stageExecutionStats.getCumulativeTotalMemory();
        userMemoryReservation += stageExecutionStats.getUserMemoryReservation().toBytes();
        totalMemoryReservation += stageExecutionStats.getTotalMemoryReservation().toBytes();
        totalScheduledTime += stageExecutionStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        totalCpuTime += stageExecutionStats.getTotalCpuTime().roundTo(MILLISECONDS);
        retriedCpuTime += computeRetriedCpuTime(stageInfo);
        totalBlockedTime += stageExecutionStats.getTotalBlockedTime().roundTo(MILLISECONDS);
        if (!stageInfo.getLatestAttemptExecutionInfo().getState().isDone()) {
            fullyBlocked &= stageExecutionStats.isFullyBlocked();
            blockedReasons.addAll(stageExecutionStats.getBlockedReasons());
        }
        totalAllocation += stageExecutionStats.getTotalAllocation().toBytes();
        if (stageInfo.getPlan().isPresent()) {
            PlanFragment plan = stageInfo.getPlan().get();
            if (!plan.getTableScanSchedulingOrder().isEmpty()) {
                rawInputDataSize += stageExecutionStats.getRawInputDataSize().toBytes();
                rawInputPositions += stageExecutionStats.getRawInputPositions();
                processedInputDataSize += stageExecutionStats.getProcessedInputDataSize().toBytes();
                processedInputPositions += stageExecutionStats.getProcessedInputPositions();
            }
            if (plan.isOutputTableWriterFragment()) {
                writtenOutputPositions += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(OperatorStats::getInputPositions).sum();
                writtenOutputLogicalDataSize += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(stats -> stats.getInputDataSize().toBytes()).sum();
                writtenOutputPhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
            } else {
                writtenIntermediatePhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
            }
        }
        stageGcStatistics.add(stageExecutionStats.getGcInfo());
        completeInfo = completeInfo && stageInfo.isFinalStageInfo();
        operatorStatsSummary.addAll(stageExecutionStats.getOperatorSummaries());
        // We prepend each metric name with the stage id to avoid merging metrics across stages.
        int stageId = stageInfo.getStageId().getId();
        stageExecutionStats.getRuntimeStats().getMetrics().forEach((name, metric) -> {
            String metricName = String.format("S%d-%s", stageId, name);
            mergedRuntimeStats.mergeMetric(metricName, metric);
        });
    }
    if (rootStage.isPresent()) {
        StageExecutionStats outputStageStats = rootStage.get().getLatestAttemptExecutionInfo().getStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
    }
    return new QueryStats(queryStateTimer.getCreateTime(), queryStateTimer.getExecutionStartTime().orElse(null), queryStateTimer.getLastHeartbeat(), queryStateTimer.getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getWaitingForPrerequisitesTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getSemanticAnalyzingTime(), queryStateTimer.getColumnAccessPermissionCheckingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, peakRunningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakTotalMemoryReservation, peakTaskUserMemory, peakTaskTotalMemory, peakNodeTotalMemory, isScheduled(rootStage), succinctDuration(totalScheduledTime, MILLISECONDS), succinctDuration(totalCpuTime, MILLISECONDS), succinctDuration(retriedCpuTime, MILLISECONDS), succinctDuration(totalBlockedTime, MILLISECONDS), fullyBlocked, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, writtenOutputPositions, succinctBytes(writtenOutputLogicalDataSize), succinctBytes(writtenOutputPhysicalDataSize), succinctBytes(writtenIntermediatePhysicalDataSize), stageGcStatistics.build(), operatorStatsSummary.build(), mergedRuntimeStats);
}

Example 9 with OperatorStats

use of com.facebook.presto.operator.OperatorStats in project presto by prestodb.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testJoinWithSelectiveBuildSide.

@Test
public void testJoinWithSelectiveBuildSide() {
    Session session = Session.builder(getSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, FeaturesConfig.JoinDistributionType.BROADCAST.name()).setSystemProperty(PUSHDOWN_SUBFIELDS_ENABLED, "false").setCatalogSessionProperty(HIVE_CATALOG, PUSHDOWN_FILTER_ENABLED, "false").build();
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    ResultWithQueryId<MaterializedResult> result = runner.executeWithQueryId(session, "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey AND orders.custkey = 1");
    assertGreaterThan(result.getResult().getRowCount(), 0);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), "lineitem");
    // Probe side may be partially scanned, depending on the drivers' scheduling:
    assertLessThanOrEqual(probeStats.getInputPositions(), countRows("lineitem"));
}