Examples with OperatorStats io.trino.operator.OperatorStats used on opensource projects

Example 16 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class BaseFailureRecoveryTest method testJoinDynamicFilteringEnabled.

@Test(invocationCount = INVOCATION_COUNT)
public void testJoinDynamicFilteringEnabled() {
    @Language("SQL") String selectQuery = "SELECT * FROM partitioned_lineitem JOIN supplier ON partitioned_lineitem.suppkey = supplier.suppkey " + "AND supplier.name = 'Supplier#000000001'";
    testSelect(selectQuery, Optional.of(enableDynamicFiltering(true)), queryId -> {
        DynamicFiltersStats dynamicFiltersStats = getDynamicFilteringStats(queryId);
        assertThat(dynamicFiltersStats.getLazyDynamicFilters()).isEqualTo(1);
        DynamicFilterDomainStats domainStats = getOnlyElement(dynamicFiltersStats.getDynamicFilterDomainStats());
        assertThat(domainStats.getSimplifiedDomain()).isEqualTo(singleValue(BIGINT, 1L).toString(getSession().toConnectorSession()));
        OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(queryId, getQualifiedTableName(PARTITIONED_LINEITEM));
        // Currently, stats from all attempts are combined.
        // Asserting on multiple of 615L as well in case the probe scan was completed twice
        assertThat(probeStats.getInputPositions()).isIn(615L, 1230L);
    });
}

Example 17 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class AbstractTestJoinQueries method assertJoinOutputPositions.

private void assertJoinOutputPositions(@Language("SQL") String sql, int expectedJoinOutputPositions) {
    ResultWithQueryId<MaterializedResult> result = getDistributedQueryRunner().executeWithQueryId(Session.builder(getSession()).setSystemProperty(JOIN_REORDERING_STRATEGY, "NONE").build(), sql);
    assertEquals(result.getResult().getMaterializedRows().get(0).getField(0), 0L);
    QueryStats stats = getDistributedQueryRunner().getCoordinator().getQueryManager().getFullQueryInfo(result.getQueryId()).getQueryStats();
    int actualJoinOutputPositions = stats.getOperatorSummaries().stream().filter(summary -> summary.getOperatorType().equals("LookupJoinOperator")).map(OperatorStats::getOutputPositions).mapToInt(Math::toIntExact).sum();
    assertEquals(actualJoinOutputPositions, expectedJoinOutputPositions);
}

Example 18 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class QueryMonitor method createQueryStatistics.

private QueryStatistics createQueryStatistics(QueryInfo queryInfo) {
    ImmutableList.Builder<String> operatorSummaries = ImmutableList.builder();
    for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) {
        operatorSummaries.add(operatorStatsCodec.toJson(summary));
    }
    Optional<StatsAndCosts> planNodeStatsAndCosts = queryInfo.getOutputStage().map(StatsAndCosts::create);
    Optional<String> serializedPlanNodeStatsAndCosts = planNodeStatsAndCosts.map(statsAndCostsCodec::toJson);
    QueryStats queryStats = queryInfo.getQueryStats();
    return new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getFailedCpuTime().toMillis()), ofMillis(queryStats.getElapsedTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getTotalScheduledTime().toMillis())), Optional.of(ofMillis(queryStats.getFailedScheduledTime().toMillis())), Optional.of(ofMillis(queryStats.getResourceWaitingTime().toMillis())), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getPlanningTime().toMillis())), Optional.of(ofMillis(queryStats.getExecutionTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getPhysicalInputDataSize().toBytes(), queryStats.getPhysicalInputPositions(), queryStats.getInternalNetworkInputDataSize().toBytes(), queryStats.getInternalNetworkInputPositions(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getFailedCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo), operatorSummaries.build(), serializedPlanNodeStatsAndCosts);
}

Example 19 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class QueryMonitor method extractPlanNodeStats.

private static void extractPlanNodeStats(StageInfo stageInfo, ImmutableMultimap.Builder<FragmentNode, OperatorStats> planNodeStats) {
    PlanFragment fragment = stageInfo.getPlan();
    if (fragment == null) {
        return;
    }
    // Note: a plan node may be mapped to multiple operators
    Map<PlanNodeId, Collection<OperatorStats>> allOperatorStats = Multimaps.index(stageInfo.getStageStats().getOperatorSummaries(), OperatorStats::getPlanNodeId).asMap();
    // Sometimes a plan node is merged with other nodes into a single operator, and in that case,
    // use the stats of the nearest parent node with stats.
    fragment.getRoot().accept(new PlanVisitor<Void, Collection<OperatorStats>>() {

        @Override
        protected Void visitPlan(PlanNode node, Collection<OperatorStats> parentStats) {
            Collection<OperatorStats> operatorStats = allOperatorStats.getOrDefault(node.getId(), parentStats);
            planNodeStats.putAll(new FragmentNode(fragment.getId(), node.getId()), operatorStats);
            for (PlanNode child : node.getSources()) {
                child.accept(this, operatorStats);
            }
            return null;
        }
    }, ImmutableList.of());
}

Example 20 with OperatorStats

use of io.trino.operator.OperatorStats in project trino by trinodb.

the class SplitMonitor method splitCompletedEvent.

private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) {
    Duration queuedTime = ofMillis(driverStats.getQueuedTime().toMillis());
    Optional<Duration> queuedTimeIfSplitRan = Optional.empty();
    if (driverStats.getStartTime() != null) {
        queuedTimeIfSplitRan = Optional.of(queuedTime);
    }
    Duration elapsedTime = ofMillis(driverStats.getElapsedTime().toMillis());
    Optional<Duration> elapsedTimeIfSplitRan = Optional.empty();
    if (driverStats.getEndTime() != null) {
        elapsedTimeIfSplitRan = Optional.of(elapsedTime);
    }
    Optional<SplitFailureInfo> splitFailureMetadata = Optional.empty();
    if (failureType != null) {
        splitFailureMetadata = Optional.of(new SplitFailureInfo(failureType, failureMessage != null ? failureMessage : ""));
    }
    Optional<String> splitCatalog = driverStats.getOperatorStats().stream().map(OperatorStats::getInfo).filter(SplitOperatorInfo.class::isInstance).map(SplitOperatorInfo.class::cast).map(info -> info.getCatalogName().getCatalogName()).findFirst();
    try {
        eventListenerManager.splitCompleted(new SplitCompletedEvent(taskId.getQueryId().toString(), taskId.getStageId().toString(), taskId.toString(), splitCatalog, driverStats.getCreateTime().toDate().toInstant(), Optional.ofNullable(driverStats.getStartTime()).map(startTime -> startTime.toDate().toInstant()), Optional.ofNullable(driverStats.getEndTime()).map(endTime -> endTime.toDate().toInstant()), new SplitStatistics(ofMillis(driverStats.getTotalCpuTime().toMillis()), elapsedTime, queuedTime, ofMillis(driverStats.getRawInputReadTime().toMillis()), driverStats.getRawInputPositions(), driverStats.getRawInputDataSize().toBytes(), queuedTimeIfSplitRan, elapsedTimeIfSplitRan), splitFailureMetadata, objectMapper.writeValueAsString(driverStats)));
    } catch (JsonProcessingException e) {
        log.error(e, "Error processing split completion event for task %s", taskId);
    }
}