use of com.facebook.presto.spi.eventlistener.QueryMetadata in project presto by prestodb.
the class QueryMonitor method queryCompletedEvent.
public void queryCompletedEvent(QueryInfo queryInfo) {
try {
Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
if (queryInfo.getFailureInfo() != null) {
FailureInfo failureInfo = queryInfo.getFailureInfo();
Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
}
ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
for (Input input : queryInfo.getInputs()) {
inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
}
QueryStats queryStats = queryInfo.getQueryStats();
Optional<QueryOutputMetadata> output = Optional.empty();
if (queryInfo.getOutput().isPresent()) {
Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
}
eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
logQueryTimeline(queryInfo);
} catch (JsonProcessingException e) {
throw Throwables.propagate(e);
}
}
use of com.facebook.presto.spi.eventlistener.QueryMetadata in project presto-audit by yahoojapan.
the class TestHelper method createNormalEvent.
public QueryCompletedEvent createNormalEvent() {
setUp();
QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3", Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"), "select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5", "FINISHED", uri, Optional.empty(), Optional.empty());
return new QueryCompletedEvent(metadata, statistics, context, ioMetadata, Optional.empty(), createTime, executionStartTime, endTime);
}
use of com.facebook.presto.spi.eventlistener.QueryMetadata in project presto by prestodb.
the class TestEventListenerWithExchangeMaterialization method testRuntimeOptimizedStagesCorrectness.
@Test
public void testRuntimeOptimizedStagesCorrectness() throws Exception {
// We expect one runtime optimized stage: 1.
int expectedEvents = 2;
QueryId queryId = runQueryAndWaitForEvents("SELECT phone, regionkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey", expectedEvents);
QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
QueryMetadata queryMetadata = queryCompletedEvent.getMetadata();
Optional<List<StageId>> runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
assertEquals(queryMetadata.getQueryId(), queryId.toString());
assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
assertTrue(runtimeOptimizedStages.isPresent());
assertEquals(runtimeOptimizedStages.get().size(), 1);
assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
// Now, the following query should not trigger runtime optimizations, so should have empty list of runtime optimized stages.
runQueryAndWaitForEvents("SELECT phone, regionkey FROM supplier INNER JOIN nation ON supplier.nationkey=nation.nationkey", expectedEvents);
queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
assertTrue(queryCompletedEvent.getMetadata().getRuntimeOptimizedStages().isEmpty());
assertFalse(runtimeOptimizedStages.isPresent());
// Now, the following query should have two optimized joins in a single stage (both on the same nationkey), therefore expect only one optimized stage: 1.
runQueryAndWaitForEvents("SELECT supplier.phone, regionkey, custkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey INNER JOIN customer ON nation.nationkey=customer.nationkey", expectedEvents);
queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
queryMetadata = queryCompletedEvent.getMetadata();
runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
assertTrue(runtimeOptimizedStages.isPresent());
assertEquals(runtimeOptimizedStages.get().size(), 1);
assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
// Now, the following query should have two runtime optimized stages: 1 and 4, corresponding to the two join operations (on regionkey and nationkey respectively).
runQueryAndWaitForEvents("WITH natreg AS (SELECT nation.regionkey, nationkey, region.name FROM region INNER JOIN nation ON nation.regionkey=region.regionkey) SELECT phone, regionkey FROM natreg INNER JOIN supplier ON supplier.nationkey=natreg.nationkey", expectedEvents);
queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
queryMetadata = queryCompletedEvent.getMetadata();
runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 2);
assertEquals(ImmutableSet.copyOf(queryMetadata.getRuntimeOptimizedStages()), ImmutableSet.of("1", "4"));
assertTrue(runtimeOptimizedStages.isPresent());
assertEquals(runtimeOptimizedStages.get().size(), 2);
assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
}
use of com.facebook.presto.spi.eventlistener.QueryMetadata in project presto-audit by yahoojapan.
the class TestHelper method createQueryWithSource.
public QueryCompletedEvent createQueryWithSource(Optional<String> source) {
setUp();
QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3", Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"), "select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5", "FINISHED", uri, Optional.empty(), Optional.empty());
QueryContext context = new QueryContext("test-user", Optional.of("principal"), Optional.of("example.com"), Optional.of("StatementClient 0.167"), Optional.of("clientInfo"), new HashSet<>(), source, Optional.of("catalog"), Optional.of("schema"), Optional.of(""), new HashMap<>(), "127.0.0.1", "0.175", "environment");
return new QueryCompletedEvent(metadata, statistics, context, ioMetadata, Optional.empty(), createTime, executionStartTime, endTime);
}
use of com.facebook.presto.spi.eventlistener.QueryMetadata in project presto-audit by yahoojapan.
the class TestHelper method createFailureEvent.
public QueryCompletedEvent createFailureEvent() {
setUp();
QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3", Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"), "select 2a", "FAILED", uri, Optional.empty(), Optional.empty());
ErrorCode errorCode = new ErrorCode(1, "SYNTAX_ERROR", ErrorType.USER_ERROR);
QueryFailureInfo failureInfo = new QueryFailureInfo(errorCode, Optional.of("com.facebook.presto.sql.parser.ParsingException"), Optional.of("line 1:15: mismatched input '0' expecting ')'"), Optional.empty(), Optional.empty(), "{json-error}");
return new QueryCompletedEvent(metadata, statistics, context, ioMetadata, Optional.of(failureInfo), createTime, executionStartTime, endTime);
}
Aggregations