Examples with OperatorStats io.trino.operator.OperatorStats used on opensource projects

Example 1 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier) {
    Optional<StageInfo> finalStageInfo = this.finalStageInfo.get();
    if (finalStageInfo.isPresent()) {
        return finalStageInfo.get();
    }
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int failedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long failedCumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long peakUserMemoryReservation = peakUserMemory.get();
    long peakRevocableMemoryReservation = peakRevocableMemory.get();
    long totalScheduledTime = 0;
    long failedScheduledTime = 0;
    long totalCpuTime = 0;
    long failedCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long failedPhysicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long failedPhysicalInputPositions = 0;
    long physicalInputReadTime = 0;
    long failedPhysicalInputReadTime = 0;
    long internalNetworkInputDataSize = 0;
    long failedInternalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long failedInternalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long failedRawInputDataSize = 0;
    long rawInputPositions = 0;
    long failedRawInputPositions = 0;
    long processedInputDataSize = 0;
    long failedProcessedInputDataSize = 0;
    long processedInputPositions = 0;
    long failedProcessedInputPositions = 0;
    long bufferedDataSize = 0;
    long outputDataSize = 0;
    long failedOutputDataSize = 0;
    long outputPositions = 0;
    long failedOutputPositions = 0;
    long physicalWrittenDataSize = 0;
    long failedPhysicalWrittenDataSize = 0;
    int fullGcCount = 0;
    int fullGcTaskCount = 0;
    int minFullGcSec = 0;
    int maxFullGcSec = 0;
    int totalFullGcSec = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        if (taskState == TaskState.FAILED) {
            failedTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        blockedDrivers += taskStats.getBlockedDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        if (taskState == TaskState.FAILED) {
            failedCumulativeUserMemory += taskStats.getCumulativeUserMemory();
        }
        long taskUserMemory = taskStats.getUserMemoryReservation().toBytes();
        long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes();
        userMemoryReservation += taskUserMemory;
        revocableMemoryReservation += taskRevocableMemory;
        totalMemoryReservation += taskUserMemory + taskRevocableMemory;
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (taskState == TaskState.FAILED) {
            failedScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
            failedCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        }
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        physicalInputDataSize += taskStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += taskStats.getPhysicalInputPositions();
        physicalInputReadTime += taskStats.getPhysicalInputReadTime().roundTo(NANOSECONDS);
        internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSize().toBytes();
        if (taskState == TaskState.FAILED) {
            failedPhysicalInputDataSize += taskStats.getPhysicalInputDataSize().toBytes();
            failedPhysicalInputPositions += taskStats.getPhysicalInputPositions();
            failedPhysicalInputReadTime += taskStats.getPhysicalInputReadTime().roundTo(NANOSECONDS);
            failedInternalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
            failedInternalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
            failedRawInputDataSize += taskStats.getRawInputDataSize().toBytes();
            failedRawInputPositions += taskStats.getRawInputPositions();
            failedProcessedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
            failedProcessedInputPositions += taskStats.getProcessedInputPositions();
            failedOutputDataSize += taskStats.getOutputDataSize().toBytes();
            failedOutputPositions += taskStats.getOutputPositions();
            failedPhysicalWrittenDataSize += taskStats.getPhysicalWrittenDataSize().toBytes();
        }
        fullGcCount += taskStats.getFullGcCount();
        fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
        int gcSec = toIntExact(taskStats.getFullGcTime().roundTo(SECONDS));
        totalFullGcSec += gcSec;
        minFullGcSec = min(minFullGcSec, gcSec);
        maxFullGcSec = max(maxFullGcSec, gcSec);
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, failedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, failedCumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(peakUserMemoryReservation), succinctBytes(peakRevocableMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(failedScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(failedCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(physicalInputDataSize), succinctBytes(failedPhysicalInputDataSize), physicalInputPositions, failedPhysicalInputPositions, succinctDuration(physicalInputReadTime, NANOSECONDS), succinctDuration(failedPhysicalInputReadTime, NANOSECONDS), succinctBytes(internalNetworkInputDataSize), succinctBytes(failedInternalNetworkInputDataSize), internalNetworkInputPositions, failedInternalNetworkInputPositions, succinctBytes(rawInputDataSize), succinctBytes(failedRawInputDataSize), rawInputPositions, failedRawInputPositions, succinctBytes(processedInputDataSize), succinctBytes(failedProcessedInputDataSize), processedInputPositions, failedProcessedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), succinctBytes(failedOutputDataSize), outputPositions, failedOutputPositions, succinctBytes(physicalWrittenDataSize), succinctBytes(failedPhysicalWrittenDataSize), new StageGcStatistics(stageId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, fragment, fragment.getPartitioning().isCoordinatorOnly(), fragment.getTypes(), stageStats, taskInfos, ImmutableList.of(), tables, failureInfo);
}

Example 2 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class TestHiveDistributedJoinQueries method testJoinWithEmptyBuildSide.

@Test
public void testJoinWithEmptyBuildSide() {
    Session session = Session.builder(getSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, BROADCAST.name()).build();
    ResultWithQueryId<MaterializedResult> result = getDistributedQueryRunner().executeWithQueryId(session, "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey AND orders.totalprice = 123.4567");
    assertEquals(result.getResult().getRowCount(), 0);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), new QualifiedObjectName(HIVE_CATALOG, "tpch", "lineitem"));
    // Probe-side is not scanned at all, due to dynamic filtering:
    assertEquals(probeStats.getInputPositions(), 0L);
    assertEquals(probeStats.getDynamicFilterSplitsProcessed(), probeStats.getTotalDrivers());
}

Example 3 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class TestDeltaLakeAnalyze method verifySplitCount.

private void verifySplitCount(QueryId queryId, long expectedCount) {
    OperatorStats operatorStats = getOperatorStats(queryId);
    assertThat(operatorStats.getTotalDrivers()).isEqualTo(expectedCount);
}

Example 4 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class QueryMonitor method getQueryIOMetadata.

private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) {
    Multimap<FragmentNode, OperatorStats> planNodeStats = extractPlanNodeStats(queryInfo);
    ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
    for (Input input : queryInfo.getInputs()) {
        // Note: input table can be mapped to multiple operators
        Collection<OperatorStats> inputTableOperatorStats = planNodeStats.get(new FragmentNode(input.getFragmentId(), input.getPlanNodeId()));
        OptionalLong physicalInputBytes = OptionalLong.empty();
        OptionalLong physicalInputPositions = OptionalLong.empty();
        if (!inputTableOperatorStats.isEmpty()) {
            physicalInputBytes = OptionalLong.of(inputTableOperatorStats.stream().map(OperatorStats::getPhysicalInputDataSize).mapToLong(DataSize::toBytes).sum());
            physicalInputPositions = OptionalLong.of(inputTableOperatorStats.stream().mapToLong(OperatorStats::getPhysicalInputPositions).sum());
        }
        inputs.add(new QueryInputMetadata(input.getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::getName).collect(Collectors.toList()), input.getConnectorInfo(), physicalInputBytes, physicalInputPositions));
    }
    Optional<QueryOutputMetadata> output = Optional.empty();
    if (queryInfo.getOutput().isPresent()) {
        Optional<TableFinishInfo> tableFinishInfo = queryInfo.getQueryStats().getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
        Optional<List<OutputColumnMetadata>> outputColumnsMetadata = queryInfo.getOutput().get().getColumns().map(columns -> columns.stream().map(column -> new OutputColumnMetadata(column.getColumn().getName(), column.getColumn().getType(), column.getSourceColumns().stream().map(Analysis.SourceColumn::getColumnDetail).collect(toImmutableSet()))).collect(toImmutableList()));
        output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), outputColumnsMetadata, tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
    }
    return new QueryIOMetadata(inputs.build(), output);
}

Example 5 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class BaseDeltaLakeConnectorSmokeTest method verifySplitCount.

private void verifySplitCount(QueryId queryId, long expectedCount) {
    OperatorStats operatorStats = getOperatorStats(queryId);
    assertThat(operatorStats.getTotalDrivers()).isEqualTo(expectedCount);
}