Examples with SplitCompletedEvent io.trino.spi.eventlistener.SplitCompletedEvent used on opensource projects

Example 1 with SplitCompletedEvent

use of io.trino.spi.eventlistener.SplitCompletedEvent in project trino by trinodb.

the class TestEventListenerWithSplits method testSplitsForNormalQuery.

@Test
public void testSplitsForNormalQuery() throws Exception {
    // We expect the following events
    // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split
    int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1;
    runQueryAndWaitForEvents("SELECT sum(linenumber) FROM lineitem", expectedEvents);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
    assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
    assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
    assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQuery(), "SELECT sum(linenumber) FROM lineitem");
    assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
    assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
    assertEquals(queryCompletedEvent.getIoMetadata().getOutput(), Optional.empty());
    assertEquals(queryCompletedEvent.getIoMetadata().getInputs().size(), 1);
    assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(getOnlyElement(queryCompletedEvent.getIoMetadata().getInputs()).getCatalogName(), "tpch");
    assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
    assertEquals(queryCompletedEvent.getStatistics().getCompletedSplits(), SPLITS_PER_NODE + 2);
    List<SplitCompletedEvent> splitCompletedEvents = generatedEvents.getSplitCompletedEvents();
    // leaf splits + aggregation split
    assertEquals(splitCompletedEvents.size(), SPLITS_PER_NODE + 2);
    // All splits must have the same query ID
    Set<String> actual = splitCompletedEvents.stream().map(SplitCompletedEvent::getQueryId).collect(toSet());
    assertEquals(actual, ImmutableSet.of(queryCompletedEvent.getMetadata().getQueryId()));
    // Sum of row count processed by all leaf stages is equal to the number of rows in the table
    long actualCompletedPositions = splitCompletedEvents.stream().filter(// filter out the root stage
    e -> !e.getStageId().endsWith(".0")).mapToLong(e -> e.getStatistics().getCompletedPositions()).sum();
    MaterializedResult result = runQueryAndWaitForEvents("SELECT count(*) FROM lineitem", expectedEvents);
    long expectedCompletedPositions = (long) result.getMaterializedRows().get(0).getField(0);
    assertEquals(actualCompletedPositions, expectedCompletedPositions);
    QueryStatistics statistics = queryCompletedEvent.getStatistics();
    // Aggregation can have memory pool usage
    assertTrue(statistics.getPeakUserMemoryBytes() >= 0);
    assertTrue(statistics.getPeakTaskUserMemory() >= 0);
    assertTrue(statistics.getPeakTaskTotalMemory() >= 0);
    assertTrue(statistics.getCumulativeMemory() >= 0);
    // Not a write query
    assertEquals(statistics.getWrittenBytes(), 0);
    assertEquals(statistics.getWrittenRows(), 0);
    assertEquals(statistics.getStageGcStatistics().size(), 2);
    // Deterministic statistics
    assertEquals(statistics.getPhysicalInputBytes(), 0);
    assertEquals(statistics.getPhysicalInputRows(), expectedCompletedPositions);
    assertEquals(statistics.getInternalNetworkBytes(), 381);
    assertEquals(statistics.getInternalNetworkRows(), 3);
    assertEquals(statistics.getTotalBytes(), 0);
    assertEquals(statistics.getOutputBytes(), 9);
    assertEquals(statistics.getOutputRows(), 1);
    assertTrue(statistics.isComplete());
    // Check only the presence because they are non-deterministic.
    assertTrue(statistics.getScheduledTime().isPresent());
    assertTrue(statistics.getResourceWaitingTime().isPresent());
    assertTrue(statistics.getAnalysisTime().isPresent());
    assertTrue(statistics.getPlanningTime().isPresent());
    assertTrue(statistics.getExecutionTime().isPresent());
    assertTrue(statistics.getPlanNodeStatsAndCosts().isPresent());
    assertTrue(statistics.getCpuTime().getSeconds() >= 0);
    assertTrue(statistics.getWallTime().getSeconds() >= 0);
    assertTrue(statistics.getCpuTimeDistribution().size() > 0);
    assertTrue(statistics.getOperatorSummaries().size() > 0);
}

Example 2 with SplitCompletedEvent

use of io.trino.spi.eventlistener.SplitCompletedEvent in project trino by trinodb.

the class TestEventListenerWithSplits method testSplitsForConstantQuery.

@Test
public void testSplitsForConstantQuery() throws Exception {
    // QueryCreated: 1, QueryCompleted: 1, Splits: 1
    runQueryAndWaitForEvents("SELECT 1", 3);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
    assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
    assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
    assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getContext().getQueryType().get(), QueryType.SELECT);
    assertEquals(queryCreatedEvent.getMetadata().getQuery(), "SELECT 1");
    assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
    assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
    assertEquals(queryCompletedEvent.getStatistics().getTotalRows(), 0L);
    assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
    assertEquals(queryCompletedEvent.getContext().getQueryType().get(), QueryType.SELECT);
    List<SplitCompletedEvent> splitCompletedEvents = generatedEvents.getSplitCompletedEvents();
    assertEquals(splitCompletedEvents.get(0).getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertEquals(splitCompletedEvents.get(0).getStatistics().getCompletedPositions(), 1);
}

Example 3 with SplitCompletedEvent

use of io.trino.spi.eventlistener.SplitCompletedEvent in project trino by trinodb.

the class SplitMonitor method splitCompletedEvent.

private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) {
    Duration queuedTime = ofMillis(driverStats.getQueuedTime().toMillis());
    Optional<Duration> queuedTimeIfSplitRan = Optional.empty();
    if (driverStats.getStartTime() != null) {
        queuedTimeIfSplitRan = Optional.of(queuedTime);
    }
    Duration elapsedTime = ofMillis(driverStats.getElapsedTime().toMillis());
    Optional<Duration> elapsedTimeIfSplitRan = Optional.empty();
    if (driverStats.getEndTime() != null) {
        elapsedTimeIfSplitRan = Optional.of(elapsedTime);
    }
    Optional<SplitFailureInfo> splitFailureMetadata = Optional.empty();
    if (failureType != null) {
        splitFailureMetadata = Optional.of(new SplitFailureInfo(failureType, failureMessage != null ? failureMessage : ""));
    }
    Optional<String> splitCatalog = driverStats.getOperatorStats().stream().map(OperatorStats::getInfo).filter(SplitOperatorInfo.class::isInstance).map(SplitOperatorInfo.class::cast).map(info -> info.getCatalogName().getCatalogName()).findFirst();
    try {
        eventListenerManager.splitCompleted(new SplitCompletedEvent(taskId.getQueryId().toString(), taskId.getStageId().toString(), taskId.toString(), splitCatalog, driverStats.getCreateTime().toDate().toInstant(), Optional.ofNullable(driverStats.getStartTime()).map(startTime -> startTime.toDate().toInstant()), Optional.ofNullable(driverStats.getEndTime()).map(endTime -> endTime.toDate().toInstant()), new SplitStatistics(ofMillis(driverStats.getTotalCpuTime().toMillis()), elapsedTime, queuedTime, ofMillis(driverStats.getRawInputReadTime().toMillis()), driverStats.getRawInputPositions(), driverStats.getRawInputDataSize().toBytes(), queuedTimeIfSplitRan, elapsedTimeIfSplitRan), splitFailureMetadata, objectMapper.writeValueAsString(driverStats)));
    } catch (JsonProcessingException e) {
        log.error(e, "Error processing split completion event for task %s", taskId);
    }
}