use of io.prestosql.spi.eventlistener.SplitCompletedEvent in project hetu-core by openlookeng.
the class SplitMonitor method splitCompletedEvent.
private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) {
Optional<Duration> timeToStart = Optional.empty();
if (driverStats.getStartTime() != null) {
timeToStart = Optional.of(ofMillis(driverStats.getStartTime().getMillis() - driverStats.getCreateTime().getMillis()));
}
Optional<Duration> timeToEnd = Optional.empty();
if (driverStats.getEndTime() != null) {
timeToEnd = Optional.of(ofMillis(driverStats.getEndTime().getMillis() - driverStats.getCreateTime().getMillis()));
}
Optional<SplitFailureInfo> splitFailureMetadata = Optional.empty();
if (failureType != null) {
splitFailureMetadata = Optional.of(new SplitFailureInfo(failureType, failureMessage != null ? failureMessage : ""));
}
try {
eventListenerManager.splitCompleted(new SplitCompletedEvent(taskId.getQueryId().toString(), taskId.getStageId().toString(), Integer.toString(taskId.getId()), driverStats.getCreateTime().toDate().toInstant(), Optional.ofNullable(driverStats.getStartTime()).map(startTime -> startTime.toDate().toInstant()), Optional.ofNullable(driverStats.getEndTime()).map(endTime -> endTime.toDate().toInstant()), new SplitStatistics(ofMillis(driverStats.getTotalCpuTime().toMillis()), ofMillis(driverStats.getElapsedTime().toMillis()), ofMillis(driverStats.getQueuedTime().toMillis()), ofMillis(driverStats.getRawInputReadTime().toMillis()), driverStats.getRawInputPositions(), driverStats.getRawInputDataSize().toBytes(), timeToStart, timeToEnd), splitFailureMetadata, objectMapper.writeValueAsString(driverStats)));
} catch (JsonProcessingException e) {
log.error(e, "Error processing split completion event for task %s", taskId);
}
}
use of io.prestosql.spi.eventlistener.SplitCompletedEvent in project hetu-core by openlookeng.
the class TestEventListener method testNormalQuery.
@Test
public void testNormalQuery() throws Exception {
// We expect the following events
// QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split
int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1;
runQueryAndWaitForEvents("SELECT sum(linenumber) FROM lineitem", expectedEvents);
QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
assertEquals(queryCreatedEvent.getMetadata().getQuery(), "SELECT sum(linenumber) FROM lineitem");
assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
assertEquals(queryCompletedEvent.getIoMetadata().getOutput(), Optional.empty());
assertEquals(queryCompletedEvent.getIoMetadata().getInputs().size(), 1);
assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
assertEquals(getOnlyElement(queryCompletedEvent.getIoMetadata().getInputs()).getCatalogName(), "tpch");
assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
assertEquals(queryCompletedEvent.getStatistics().getCompletedSplits(), SPLITS_PER_NODE + 2);
List<SplitCompletedEvent> splitCompletedEvents = generatedEvents.getSplitCompletedEvents();
// leaf splits + aggregation split
assertEquals(splitCompletedEvents.size(), SPLITS_PER_NODE + 2);
// All splits must have the same query ID
Set<String> actual = splitCompletedEvents.stream().map(SplitCompletedEvent::getQueryId).collect(toSet());
assertEquals(actual, ImmutableSet.of(queryCompletedEvent.getMetadata().getQueryId()));
// Sum of row count processed by all leaf stages is equal to the number of rows in the table
long actualCompletedPositions = splitCompletedEvents.stream().filter(// filter out the root stage
e -> !e.getStageId().endsWith(".0")).mapToLong(e -> e.getStatistics().getCompletedPositions()).sum();
MaterializedResult result = runQueryAndWaitForEvents("SELECT count(*) FROM lineitem", expectedEvents);
long expectedCompletedPositions = (long) result.getMaterializedRows().get(0).getField(0);
assertEquals(actualCompletedPositions, expectedCompletedPositions);
assertEquals(queryCompletedEvent.getStatistics().getTotalRows(), expectedCompletedPositions);
}
use of io.prestosql.spi.eventlistener.SplitCompletedEvent in project hetu-core by openlookeng.
the class TestEventListener method testConstantQuery.
@Test
public void testConstantQuery() throws Exception {
// QueryCreated: 1, QueryCompleted: 1, Splits: 1
runQueryAndWaitForEvents("SELECT 1", 3);
QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
assertEquals(queryCreatedEvent.getMetadata().getQuery(), "SELECT 1");
assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
assertEquals(queryCompletedEvent.getStatistics().getTotalRows(), 0L);
assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
List<SplitCompletedEvent> splitCompletedEvents = generatedEvents.getSplitCompletedEvents();
assertEquals(splitCompletedEvents.get(0).getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
assertEquals(splitCompletedEvents.get(0).getStatistics().getCompletedPositions(), 1);
}
Aggregations