Examples with OperatorStats io.trino.operator.OperatorStats used on opensource projects

Example 21 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class TestMemoryTracking method assertStats.

private void assertStats(List<OperatorStats> nestedOperatorStats, DriverStats driverStats, PipelineStats pipelineStats, TaskStats taskStats, long expectedUserMemory, long expectedRevocableMemory) {
    OperatorStats operatorStats = getOnlyElement(nestedOperatorStats);
    assertEquals(operatorStats.getUserMemoryReservation().toBytes(), expectedUserMemory);
    assertEquals(driverStats.getUserMemoryReservation().toBytes(), expectedUserMemory);
    assertEquals(pipelineStats.getUserMemoryReservation().toBytes(), expectedUserMemory);
    assertEquals(taskStats.getUserMemoryReservation().toBytes(), expectedUserMemory);
    assertEquals(operatorStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory);
    assertEquals(driverStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory);
    assertEquals(pipelineStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory);
    assertEquals(taskStats.getRevocableMemoryReservation().toBytes(), expectedRevocableMemory);
}

Example 22 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class PlanNodeStatsSummarizer method getPlanNodeStats.

private static List<PlanNodeStats> getPlanNodeStats(TaskStats taskStats) {
    // Best effort to reconstruct the plan nodes from operators.
    // Because stats are collected separately from query execution,
    // it's possible that some or all of them are missing or out of date.
    // For example, a LIMIT clause can cause a query to finish before stats
    // are collected from the leaf stages.
    Set<PlanNodeId> planNodeIds = new HashSet<>();
    Map<PlanNodeId, Long> planNodeInputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeInputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputPositions = new HashMap<>();
    Map<PlanNodeId, Long> planNodeOutputBytes = new HashMap<>();
    Map<PlanNodeId, Long> planNodeSpilledDataSize = new HashMap<>();
    Map<PlanNodeId, Long> planNodeScheduledMillis = new HashMap<>();
    Map<PlanNodeId, Long> planNodeCpuMillis = new HashMap<>();
    Map<PlanNodeId, Map<String, BasicOperatorStats>> basicOperatorStats = new HashMap<>();
    Map<PlanNodeId, Map<String, OperatorHashCollisionsStats>> operatorHashCollisionsStats = new HashMap<>();
    Map<PlanNodeId, WindowOperatorStats> windowNodeStats = new HashMap<>();
    for (PipelineStats pipelineStats : taskStats.getPipelines()) {
        // Due to eventual consistently collected stats, these could be empty
        if (pipelineStats.getOperatorSummaries().isEmpty()) {
            continue;
        }
        Set<PlanNodeId> processedNodes = new HashSet<>();
        PlanNodeId inputPlanNode = pipelineStats.getOperatorSummaries().iterator().next().getPlanNodeId();
        PlanNodeId outputPlanNode = getLast(pipelineStats.getOperatorSummaries()).getPlanNodeId();
        // Gather input statistics
        for (OperatorStats operatorStats : pipelineStats.getOperatorSummaries()) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            planNodeIds.add(planNodeId);
            long scheduledMillis = operatorStats.getAddInputWall().toMillis() + operatorStats.getGetOutputWall().toMillis() + operatorStats.getFinishWall().toMillis();
            planNodeScheduledMillis.merge(planNodeId, scheduledMillis, Long::sum);
            long cpuMillis = operatorStats.getAddInputCpu().toMillis() + operatorStats.getGetOutputCpu().toMillis() + operatorStats.getFinishCpu().toMillis();
            planNodeCpuMillis.merge(planNodeId, cpuMillis, Long::sum);
            // A pipeline like hash build before join might link to another "internal" pipelines which provide actual input for this plan node
            if (operatorStats.getPlanNodeId().equals(inputPlanNode) && !pipelineStats.isInputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            basicOperatorStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new BasicOperatorStats(operatorStats.getTotalDrivers(), operatorStats.getInputPositions(), operatorStats.getSumSquaredInputPositions(), operatorStats.getMetrics(), operatorStats.getConnectorMetrics())), (map1, map2) -> mergeMaps(map1, map2, BasicOperatorStats::merge));
            planNodeInputPositions.merge(planNodeId, operatorStats.getInputPositions(), Long::sum);
            planNodeInputBytes.merge(planNodeId, operatorStats.getInputDataSize().toBytes(), Long::sum);
            planNodeSpilledDataSize.merge(planNodeId, operatorStats.getSpilledDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
        // Gather output statistics
        processedNodes.clear();
        for (OperatorStats operatorStats : reverse(pipelineStats.getOperatorSummaries())) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            // An "internal" pipeline like a hash build, links to another pipeline which is the actual output for this plan node
            if (operatorStats.getPlanNodeId().equals(outputPlanNode) && !pipelineStats.isOutputPipeline()) {
                continue;
            }
            if (processedNodes.contains(planNodeId)) {
                continue;
            }
            planNodeOutputPositions.merge(planNodeId, operatorStats.getOutputPositions(), Long::sum);
            planNodeOutputBytes.merge(planNodeId, operatorStats.getOutputDataSize().toBytes(), Long::sum);
            processedNodes.add(planNodeId);
        }
        // Gather auxiliary statistics
        for (OperatorStats operatorStats : pipelineStats.getOperatorSummaries()) {
            PlanNodeId planNodeId = operatorStats.getPlanNodeId();
            if (operatorStats.getInfo() instanceof HashCollisionsInfo) {
                HashCollisionsInfo hashCollisionsInfo = (HashCollisionsInfo) operatorStats.getInfo();
                operatorHashCollisionsStats.merge(planNodeId, ImmutableMap.of(operatorStats.getOperatorType(), new OperatorHashCollisionsStats(hashCollisionsInfo.getWeightedHashCollisions(), hashCollisionsInfo.getWeightedSumSquaredHashCollisions(), hashCollisionsInfo.getWeightedExpectedHashCollisions(), operatorStats.getInputPositions())), (map1, map2) -> mergeMaps(map1, map2, OperatorHashCollisionsStats::merge));
            }
            // The only statistics we have for Window Functions are very low level, thus displayed only in VERBOSE mode
            if (operatorStats.getInfo() instanceof WindowInfo) {
                WindowInfo windowInfo = (WindowInfo) operatorStats.getInfo();
                windowNodeStats.merge(planNodeId, WindowOperatorStats.create(windowInfo), WindowOperatorStats::mergeWith);
            }
        }
    }
    List<PlanNodeStats> stats = new ArrayList<>();
    for (PlanNodeId planNodeId : planNodeIds) {
        if (!planNodeInputPositions.containsKey(planNodeId)) {
            continue;
        }
        PlanNodeStats nodeStats;
        // It's possible there will be no output stats because all the pipelines that we observed were non-output.
        // For example in a query like SELECT * FROM a JOIN b ON c = d LIMIT 1
        // It's possible to observe stats after the build starts, but before the probe does
        // and therefore only have scheduled time, but no output stats
        long outputPositions = planNodeOutputPositions.getOrDefault(planNodeId, 0L);
        if (operatorHashCollisionsStats.containsKey(planNodeId)) {
            nodeStats = new HashCollisionPlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctBytes(planNodeInputBytes.get(planNodeId)), outputPositions, succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), basicOperatorStats.get(planNodeId), operatorHashCollisionsStats.get(planNodeId));
        } else if (windowNodeStats.containsKey(planNodeId)) {
            nodeStats = new WindowPlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctBytes(planNodeInputBytes.get(planNodeId)), outputPositions, succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), basicOperatorStats.get(planNodeId), windowNodeStats.get(planNodeId));
        } else {
            nodeStats = new PlanNodeStats(planNodeId, new Duration(planNodeScheduledMillis.get(planNodeId), MILLISECONDS), new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctBytes(planNodeInputBytes.get(planNodeId)), outputPositions, succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), basicOperatorStats.get(planNodeId));
        }
        stats.add(nodeStats);
    }
    return stats;
}

Example 23 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class QueryStateMachine method getQueryStats.

private QueryStats getQueryStats(Optional<StageInfo> rootStage) {
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int failedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long failedCumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long failedScheduledTime = 0;
    long totalCpuTime = 0;
    long failedCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long failedPhysicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long failedPhysicalInputPositions = 0;
    long physicalInputReadTime = 0;
    long failedPhysicalInputReadTime = 0;
    long internalNetworkInputDataSize = 0;
    long failedInternalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long failedInternalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long failedRawInputDataSize = 0;
    long rawInputPositions = 0;
    long failedRawInputPositions = 0;
    long processedInputDataSize = 0;
    long failedProcessedInputDataSize = 0;
    long processedInputPositions = 0;
    long failedProcessedInputPositions = 0;
    long outputDataSize = 0;
    long failedOutputDataSize = 0;
    long outputPositions = 0;
    long failedOutputPositions = 0;
    long physicalWrittenDataSize = 0;
    long failedPhysicalWrittenDataSize = 0;
    ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageStats stageStats = stageInfo.getStageStats();
        totalTasks += stageStats.getTotalTasks();
        runningTasks += stageStats.getRunningTasks();
        completedTasks += stageStats.getCompletedTasks();
        failedTasks += stageStats.getFailedTasks();
        totalDrivers += stageStats.getTotalDrivers();
        queuedDrivers += stageStats.getQueuedDrivers();
        runningDrivers += stageStats.getRunningDrivers();
        blockedDrivers += stageStats.getBlockedDrivers();
        completedDrivers += stageStats.getCompletedDrivers();
        cumulativeUserMemory += stageStats.getCumulativeUserMemory();
        failedCumulativeUserMemory += stageStats.getFailedCumulativeUserMemory();
        userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
        revocableMemoryReservation += stageStats.getRevocableMemoryReservation().toBytes();
        totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
        totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        failedScheduledTime += stageStats.getFailedScheduledTime().roundTo(MILLISECONDS);
        totalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS);
        failedCpuTime += stageStats.getFailedCpuTime().roundTo(MILLISECONDS);
        totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(MILLISECONDS);
        if (!stageInfo.getState().isDone()) {
            fullyBlocked &= stageStats.isFullyBlocked();
            blockedReasons.addAll(stageStats.getBlockedReasons());
        }
        physicalInputDataSize += stageStats.getPhysicalInputDataSize().toBytes();
        failedPhysicalInputDataSize += stageStats.getFailedPhysicalInputDataSize().toBytes();
        physicalInputPositions += stageStats.getPhysicalInputPositions();
        failedPhysicalInputPositions += stageStats.getFailedPhysicalInputPositions();
        physicalInputReadTime += stageStats.getPhysicalInputReadTime().roundTo(MILLISECONDS);
        failedPhysicalInputReadTime += stageStats.getFailedPhysicalInputReadTime().roundTo(MILLISECONDS);
        internalNetworkInputDataSize += stageStats.getInternalNetworkInputDataSize().toBytes();
        failedInternalNetworkInputDataSize += stageStats.getFailedInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += stageStats.getInternalNetworkInputPositions();
        failedInternalNetworkInputPositions += stageStats.getFailedInternalNetworkInputPositions();
        PlanFragment plan = stageInfo.getPlan();
        if (plan != null && plan.getPartitionedSourceNodes().stream().anyMatch(TableScanNode.class::isInstance)) {
            rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
            failedRawInputDataSize += stageStats.getFailedRawInputDataSize().toBytes();
            rawInputPositions += stageStats.getRawInputPositions();
            failedRawInputPositions += stageStats.getFailedRawInputPositions();
            processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
            failedProcessedInputDataSize += stageStats.getFailedProcessedInputDataSize().toBytes();
            processedInputPositions += stageStats.getProcessedInputPositions();
            failedProcessedInputPositions += stageStats.getFailedProcessedInputPositions();
        }
        physicalWrittenDataSize += stageStats.getPhysicalWrittenDataSize().toBytes();
        failedPhysicalWrittenDataSize += stageStats.getFailedPhysicalWrittenDataSize().toBytes();
        stageGcStatistics.add(stageStats.getGcInfo());
        completeInfo = completeInfo && stageInfo.isCompleteInfo();
        operatorStatsSummary.addAll(stageInfo.getStageStats().getOperatorSummaries());
    }
    if (rootStage.isPresent()) {
        StageStats outputStageStats = rootStage.get().getStageStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        failedOutputDataSize += outputStageStats.getFailedOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
        failedOutputPositions += outputStageStats.getFailedOutputPositions();
    }
    boolean isScheduled = isScheduled(rootStage);
    return new QueryStats(queryStateTimer.getCreateTime(), getExecutionStartTime().orElse(null), getLastHeartbeat(), getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, completedTasks, failedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, failedCumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(getPeakUserMemoryInBytes()), succinctBytes(getPeakRevocableMemoryInBytes()), succinctBytes(getPeakTotalMemoryInBytes()), succinctBytes(getPeakTaskUserMemory()), succinctBytes(getPeakTaskRevocableMemory()), succinctBytes(getPeakTaskTotalMemory()), isScheduled, new Duration(totalScheduledTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(failedScheduledTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(failedCpuTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), fullyBlocked, blockedReasons, succinctBytes(physicalInputDataSize), succinctBytes(failedPhysicalInputDataSize), physicalInputPositions, failedPhysicalInputPositions, new Duration(physicalInputReadTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(failedPhysicalInputReadTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), succinctBytes(internalNetworkInputDataSize), succinctBytes(failedInternalNetworkInputDataSize), internalNetworkInputPositions, failedInternalNetworkInputPositions, succinctBytes(rawInputDataSize), succinctBytes(failedRawInputDataSize), rawInputPositions, failedRawInputPositions, succinctBytes(processedInputDataSize), succinctBytes(failedProcessedInputDataSize), processedInputPositions, failedProcessedInputPositions, succinctBytes(outputDataSize), succinctBytes(failedOutputDataSize), outputPositions, failedOutputPositions, succinctBytes(physicalWrittenDataSize), succinctBytes(failedPhysicalWrittenDataSize), stageGcStatistics.build(), getDynamicFiltersStats(), operatorStatsSummary.build());
}

Example 24 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class BaseDynamicPartitionPruningTest method testJoinWithEmptyBuildSide.

@Test(timeOut = 30_000)
public void testJoinWithEmptyBuildSide() {
    @Language("SQL") String selectQuery = "SELECT * FROM partitioned_lineitem JOIN supplier ON partitioned_lineitem.suppkey = supplier.suppkey AND supplier.name = 'abc'";
    ResultWithQueryId<MaterializedResult> result = getDistributedQueryRunner().executeWithQueryId(getSession(), selectQuery);
    MaterializedResult expected = computeActual(withDynamicFilteringDisabled(), selectQuery);
    assertEqualsIgnoreOrder(result.getResult(), expected);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), getQualifiedTableName(PARTITIONED_LINEITEM));
    assertEquals(probeStats.getInputPositions(), 0L);
    DynamicFiltersStats dynamicFiltersStats = getDynamicFilteringStats(result.getQueryId());
    assertEquals(dynamicFiltersStats.getTotalDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getLazyDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getReplicatedDynamicFilters(), 0L);
    assertEquals(dynamicFiltersStats.getDynamicFiltersCompleted(), 1L);
    DynamicFilterDomainStats domainStats = getOnlyElement(dynamicFiltersStats.getDynamicFilterDomainStats());
    assertEquals(domainStats.getSimplifiedDomain(), none(BIGINT).toString(getSession().toConnectorSession()));
    assertTrue(domainStats.getCollectionDuration().isPresent());
}

Example 25 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class BaseDynamicPartitionPruningTest method testSemiJoinWithSelectiveBuildSide.

@Test(timeOut = 30_000)
public void testSemiJoinWithSelectiveBuildSide() {
    @Language("SQL") String selectQuery = "SELECT * FROM partitioned_lineitem WHERE suppkey IN (SELECT suppkey FROM supplier WHERE name = 'Supplier#000000001')";
    ResultWithQueryId<MaterializedResult> result = getDistributedQueryRunner().executeWithQueryId(getSession(), selectQuery);
    MaterializedResult expected = computeActual(withDynamicFilteringDisabled(), selectQuery);
    assertEqualsIgnoreOrder(result.getResult(), expected);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), getQualifiedTableName(PARTITIONED_LINEITEM));
    // Probe-side is partially scanned
    assertEquals(probeStats.getInputPositions(), 615L);
    DynamicFiltersStats dynamicFiltersStats = getDynamicFilteringStats(result.getQueryId());
    assertEquals(dynamicFiltersStats.getTotalDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getLazyDynamicFilters(), 1L);
    assertEquals(dynamicFiltersStats.getReplicatedDynamicFilters(), 0L);
    assertEquals(dynamicFiltersStats.getDynamicFiltersCompleted(), 1L);
    DynamicFilterDomainStats domainStats = getOnlyElement(dynamicFiltersStats.getDynamicFilterDomainStats());
    assertEquals(domainStats.getSimplifiedDomain(), singleValue(BIGINT, 1L).toString(getSession().toConnectorSession()));
}