Examples with QueryStatistics io.trino.spi.eventlistener.QueryStatistics used on opensource projects

Example 1 with QueryStatistics

use of io.trino.spi.eventlistener.QueryStatistics in project trino by trinodb.

the class QueryMonitor method queryImmediateFailureEvent.

public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailureInfo failure) {
    eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getUpdateType(), queryInfo.getPreparedQuery(), queryInfo.getState().toString(), ImmutableList.of(), ImmutableList.of(), queryInfo.getSelf(), Optional.empty(), Optional.empty()), new QueryStatistics(ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(queryInfo.getQueryStats().getQueuedTime().toMillis()), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ImmutableList.of(), 0, true, ImmutableList.of(), ImmutableList.of(), Optional.empty()), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId(), queryInfo.getQueryType()), new QueryIOMetadata(ImmutableList.of(), Optional.empty()), createQueryFailureInfo(failure, Optional.empty()), ImmutableList.of(), ofEpochMilli(queryInfo.getQueryStats().getCreateTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis())));
    logQueryTimeline(queryInfo);
}

Example 2 with QueryStatistics

use of io.trino.spi.eventlistener.QueryStatistics in project trino by trinodb.

the class TestEventListenerWithSplits method testSplitsForNormalQuery.

@Test
public void testSplitsForNormalQuery() throws Exception {
    // We expect the following events
    // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split
    int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1;
    runQueryAndWaitForEvents("SELECT sum(linenumber) FROM lineitem", expectedEvents);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
    assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
    assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
    assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQuery(), "SELECT sum(linenumber) FROM lineitem");
    assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
    assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
    assertEquals(queryCompletedEvent.getIoMetadata().getOutput(), Optional.empty());
    assertEquals(queryCompletedEvent.getIoMetadata().getInputs().size(), 1);
    assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(getOnlyElement(queryCompletedEvent.getIoMetadata().getInputs()).getCatalogName(), "tpch");
    assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
    assertEquals(queryCompletedEvent.getStatistics().getCompletedSplits(), SPLITS_PER_NODE + 2);
    List<SplitCompletedEvent> splitCompletedEvents = generatedEvents.getSplitCompletedEvents();
    // leaf splits + aggregation split
    assertEquals(splitCompletedEvents.size(), SPLITS_PER_NODE + 2);
    // All splits must have the same query ID
    Set<String> actual = splitCompletedEvents.stream().map(SplitCompletedEvent::getQueryId).collect(toSet());
    assertEquals(actual, ImmutableSet.of(queryCompletedEvent.getMetadata().getQueryId()));
    // Sum of row count processed by all leaf stages is equal to the number of rows in the table
    long actualCompletedPositions = splitCompletedEvents.stream().filter(// filter out the root stage
    e -> !e.getStageId().endsWith(".0")).mapToLong(e -> e.getStatistics().getCompletedPositions()).sum();
    MaterializedResult result = runQueryAndWaitForEvents("SELECT count(*) FROM lineitem", expectedEvents);
    long expectedCompletedPositions = (long) result.getMaterializedRows().get(0).getField(0);
    assertEquals(actualCompletedPositions, expectedCompletedPositions);
    QueryStatistics statistics = queryCompletedEvent.getStatistics();
    // Aggregation can have memory pool usage
    assertTrue(statistics.getPeakUserMemoryBytes() >= 0);
    assertTrue(statistics.getPeakTaskUserMemory() >= 0);
    assertTrue(statistics.getPeakTaskTotalMemory() >= 0);
    assertTrue(statistics.getCumulativeMemory() >= 0);
    // Not a write query
    assertEquals(statistics.getWrittenBytes(), 0);
    assertEquals(statistics.getWrittenRows(), 0);
    assertEquals(statistics.getStageGcStatistics().size(), 2);
    // Deterministic statistics
    assertEquals(statistics.getPhysicalInputBytes(), 0);
    assertEquals(statistics.getPhysicalInputRows(), expectedCompletedPositions);
    assertEquals(statistics.getInternalNetworkBytes(), 381);
    assertEquals(statistics.getInternalNetworkRows(), 3);
    assertEquals(statistics.getTotalBytes(), 0);
    assertEquals(statistics.getOutputBytes(), 9);
    assertEquals(statistics.getOutputRows(), 1);
    assertTrue(statistics.isComplete());
    // Check only the presence because they are non-deterministic.
    assertTrue(statistics.getScheduledTime().isPresent());
    assertTrue(statistics.getResourceWaitingTime().isPresent());
    assertTrue(statistics.getAnalysisTime().isPresent());
    assertTrue(statistics.getPlanningTime().isPresent());
    assertTrue(statistics.getExecutionTime().isPresent());
    assertTrue(statistics.getPlanNodeStatsAndCosts().isPresent());
    assertTrue(statistics.getCpuTime().getSeconds() >= 0);
    assertTrue(statistics.getWallTime().getSeconds() >= 0);
    assertTrue(statistics.getCpuTimeDistribution().size() > 0);
    assertTrue(statistics.getOperatorSummaries().size() > 0);
}

Example 3 with QueryStatistics

use of io.trino.spi.eventlistener.QueryStatistics in project trino by trinodb.

the class QueryMonitor method createQueryStatistics.

private QueryStatistics createQueryStatistics(QueryInfo queryInfo) {
    ImmutableList.Builder<String> operatorSummaries = ImmutableList.builder();
    for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) {
        operatorSummaries.add(operatorStatsCodec.toJson(summary));
    }
    Optional<StatsAndCosts> planNodeStatsAndCosts = queryInfo.getOutputStage().map(StatsAndCosts::create);
    Optional<String> serializedPlanNodeStatsAndCosts = planNodeStatsAndCosts.map(statsAndCostsCodec::toJson);
    QueryStats queryStats = queryInfo.getQueryStats();
    return new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getFailedCpuTime().toMillis()), ofMillis(queryStats.getElapsedTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getTotalScheduledTime().toMillis())), Optional.of(ofMillis(queryStats.getFailedScheduledTime().toMillis())), Optional.of(ofMillis(queryStats.getResourceWaitingTime().toMillis())), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getPlanningTime().toMillis())), Optional.of(ofMillis(queryStats.getExecutionTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getPhysicalInputDataSize().toBytes(), queryStats.getPhysicalInputPositions(), queryStats.getInternalNetworkInputDataSize().toBytes(), queryStats.getInternalNetworkInputPositions(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getFailedCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo), operatorSummaries.build(), serializedPlanNodeStatsAndCosts);
}

Example 4 with QueryStatistics

use of io.trino.spi.eventlistener.QueryStatistics in project trino by trinodb.

the class TestEventListenerBasic method testOutputStats.

@Test
public void testOutputStats() throws Exception {
    MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", 2);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    QueryStats queryStats = getDistributedQueryRunner().getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(result.getRowCount(), queryStats.getOutputPositions());
    assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows());
    runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", 2);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryStats = getDistributedQueryRunner().getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(1L, queryStats.getOutputPositions());
    assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows());
    // Ensure the proper conversion in QueryMonitor#createQueryStatistics
    QueryStatistics statistics = queryCompletedEvent.getStatistics();
    assertEquals(statistics.getCpuTime().toMillis(), queryStats.getTotalCpuTime().toMillis());
    assertEquals(statistics.getWallTime().toMillis(), queryStats.getElapsedTime().toMillis());
    assertEquals(statistics.getQueuedTime().toMillis(), queryStats.getQueuedTime().toMillis());
    assertEquals(statistics.getScheduledTime().get().toMillis(), queryStats.getTotalScheduledTime().toMillis());
    assertEquals(statistics.getResourceWaitingTime().get().toMillis(), queryStats.getResourceWaitingTime().toMillis());
    assertEquals(statistics.getAnalysisTime().get().toMillis(), queryStats.getAnalysisTime().toMillis());
    assertEquals(statistics.getPlanningTime().get().toMillis(), queryStats.getPlanningTime().toMillis());
    assertEquals(statistics.getExecutionTime().get().toMillis(), queryStats.getExecutionTime().toMillis());
    assertEquals(statistics.getPeakUserMemoryBytes(), queryStats.getPeakUserMemoryReservation().toBytes());
    assertEquals(statistics.getPeakTaskUserMemory(), queryStats.getPeakTaskUserMemory().toBytes());
    assertEquals(statistics.getPeakTaskTotalMemory(), queryStats.getPeakTaskTotalMemory().toBytes());
    assertEquals(statistics.getPhysicalInputBytes(), queryStats.getPhysicalInputDataSize().toBytes());
    assertEquals(statistics.getPhysicalInputRows(), queryStats.getPhysicalInputPositions());
    assertEquals(statistics.getInternalNetworkBytes(), queryStats.getInternalNetworkInputDataSize().toBytes());
    assertEquals(statistics.getInternalNetworkRows(), queryStats.getInternalNetworkInputPositions());
    assertEquals(statistics.getTotalBytes(), queryStats.getRawInputDataSize().toBytes());
    assertEquals(statistics.getTotalRows(), queryStats.getRawInputPositions());
    assertEquals(statistics.getOutputBytes(), queryStats.getOutputDataSize().toBytes());
    assertEquals(statistics.getOutputRows(), queryStats.getOutputPositions());
    assertEquals(statistics.getWrittenBytes(), queryStats.getLogicalWrittenDataSize().toBytes());
    assertEquals(statistics.getWrittenRows(), queryStats.getWrittenPositions());
    assertEquals(statistics.getCumulativeMemory(), queryStats.getCumulativeUserMemory());
    assertEquals(statistics.getStageGcStatistics(), queryStats.getStageGcStatistics());
    assertEquals(statistics.getCompletedSplits(), queryStats.getCompletedDrivers());
}