Search in sources :

Example 1 with QueryCompletedEvent

use of com.facebook.presto.spi.eventlistener.QueryCompletedEvent in project presto by prestodb.

the class QueryMonitor method queryCompletedEvent.

public void queryCompletedEvent(QueryInfo queryInfo) {
    try {
        Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
        if (queryInfo.getFailureInfo() != null) {
            FailureInfo failureInfo = queryInfo.getFailureInfo();
            Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
            queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
        }
        ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
        for (Input input : queryInfo.getInputs()) {
            inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
        }
        QueryStats queryStats = queryInfo.getQueryStats();
        Optional<QueryOutputMetadata> output = Optional.empty();
        if (queryInfo.getOutput().isPresent()) {
            Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
            output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
        }
        eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
        logQueryTimeline(queryInfo);
    } catch (JsonProcessingException e) {
        throw Throwables.propagate(e);
    }
}
Also used : TaskStats(com.facebook.presto.operator.TaskStats) QueryContext(com.facebook.presto.spi.eventlistener.QueryContext) NodeInfo(io.airlift.node.NodeInfo) TaskState(com.facebook.presto.execution.TaskState) NodeVersion(com.facebook.presto.client.NodeVersion) TableFinishInfo(com.facebook.presto.operator.TableFinishInfo) EventListenerManager(com.facebook.presto.eventlistener.EventListenerManager) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) OperatorStats(com.facebook.presto.operator.OperatorStats) StageInfo(com.facebook.presto.execution.StageInfo) Instant.ofEpochMilli(java.time.Instant.ofEpochMilli) Duration(java.time.Duration) Map(java.util.Map) QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) TransactionId(com.facebook.presto.transaction.TransactionId) SplitStatistics(com.facebook.presto.spi.eventlistener.SplitStatistics) Column(com.facebook.presto.execution.Column) ImmutableMap(com.google.common.collect.ImmutableMap) SplitCompletedEvent(com.facebook.presto.spi.eventlistener.SplitCompletedEvent) Collectors(java.util.stream.Collectors) List(java.util.List) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) Input(com.facebook.presto.execution.Input) QueryInfo(com.facebook.presto.execution.QueryInfo) Optional(java.util.Optional) Math.max(java.lang.Math.max) ConnectorId(com.facebook.presto.connector.ConnectorId) Duration.ofMillis(java.time.Duration.ofMillis) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) JsonCodec(io.airlift.json.JsonCodec) DriverStats(com.facebook.presto.operator.DriverStats) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) Logger(io.airlift.log.Logger) Inject(javax.inject.Inject) ImmutableList(com.google.common.collect.ImmutableList) FailureInfo(com.facebook.presto.client.FailureInfo) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) Nullable(javax.annotation.Nullable) QueryOutputMetadata(com.facebook.presto.spi.eventlistener.QueryOutputMetadata) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) Throwables(com.google.common.base.Throwables) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) QueryInputMetadata(com.facebook.presto.spi.eventlistener.QueryInputMetadata) QueryStats(com.facebook.presto.execution.QueryStats) SplitFailureInfo(com.facebook.presto.spi.eventlistener.SplitFailureInfo) TaskId(com.facebook.presto.execution.TaskId) QueryCreatedEvent(com.facebook.presto.spi.eventlistener.QueryCreatedEvent) TaskInfo(com.facebook.presto.execution.TaskInfo) TableFinishInfo(com.facebook.presto.operator.TableFinishInfo) QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) ImmutableList(com.google.common.collect.ImmutableList) QueryOutputMetadata(com.facebook.presto.spi.eventlistener.QueryOutputMetadata) QueryContext(com.facebook.presto.spi.eventlistener.QueryContext) QueryInputMetadata(com.facebook.presto.spi.eventlistener.QueryInputMetadata) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) TaskInfo(com.facebook.presto.execution.TaskInfo) Input(com.facebook.presto.execution.Input) QueryStats(com.facebook.presto.execution.QueryStats) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) FailureInfo(com.facebook.presto.client.FailureInfo) SplitFailureInfo(com.facebook.presto.spi.eventlistener.SplitFailureInfo) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException)

Example 2 with QueryCompletedEvent

use of com.facebook.presto.spi.eventlistener.QueryCompletedEvent in project presto-audit by yahoojapan.

the class TestHelper method createNormalEvent.

public QueryCompletedEvent createNormalEvent() {
    setUp();
    QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3", Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"), "select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5", "FINISHED", uri, Optional.empty(), Optional.empty());
    return new QueryCompletedEvent(metadata, statistics, context, ioMetadata, Optional.empty(), createTime, executionStartTime, endTime);
}
Also used : QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent)

Example 3 with QueryCompletedEvent

use of com.facebook.presto.spi.eventlistener.QueryCompletedEvent in project presto by prestodb.

the class TestEventListener method testPrepareAndExecute.

@Test
public void testPrepareAndExecute() throws Exception {
    String selectQuery = "SELECT count(*) FROM lineitem WHERE shipmode = ?";
    String prepareQuery = "PREPARE stmt FROM " + selectQuery;
    // QueryCreated: 1, QueryCompleted: 1, Splits: 0
    runQueryAndWaitForEvents(prepareQuery, 2);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
    assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
    assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
    assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQuery(), prepareQuery);
    assertFalse(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
    assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
    assertEquals(queryCompletedEvent.getIoMetadata().getOutput(), Optional.empty());
    // Prepare has no inputs
    assertEquals(queryCompletedEvent.getIoMetadata().getInputs().size(), 0);
    assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertFalse(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
    // Prepare has no splits
    assertEquals(queryCompletedEvent.getStatistics().getCompletedSplits(), 0);
    // Add prepared statement to a new session to eliminate any impact on other tests in this suite.
    Session sessionWithPrepare = Session.builder(session).addPreparedStatement("stmt", selectQuery).build();
    // We expect the following events
    // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split
    int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1;
    runQueryAndWaitForEvents("EXECUTE stmt USING 'SHIP'", expectedEvents, sessionWithPrepare);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    assertEquals(queryCreatedEvent.getContext().getServerVersion(), "testversion");
    assertEquals(queryCreatedEvent.getContext().getServerAddress(), "127.0.0.1");
    assertEquals(queryCreatedEvent.getContext().getEnvironment(), "testing");
    assertEquals(queryCreatedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(queryCreatedEvent.getMetadata().getQuery(), "EXECUTE stmt USING 'SHIP'");
    assertTrue(queryCreatedEvent.getMetadata().getPreparedQuery().isPresent());
    assertEquals(queryCreatedEvent.getMetadata().getPreparedQuery().get(), selectQuery);
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    assertTrue(queryCompletedEvent.getContext().getResourceGroupId().isPresent());
    assertEquals(queryCompletedEvent.getContext().getResourceGroupId().get(), createResourceGroupId("global", "user-user"));
    assertEquals(queryCompletedEvent.getIoMetadata().getOutput(), Optional.empty());
    assertEquals(queryCompletedEvent.getIoMetadata().getInputs().size(), 1);
    assertEquals(queryCompletedEvent.getContext().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    assertEquals(getOnlyElement(queryCompletedEvent.getIoMetadata().getInputs()).getCatalogName(), "tpch");
    assertEquals(queryCreatedEvent.getMetadata().getQueryId(), queryCompletedEvent.getMetadata().getQueryId());
    assertTrue(queryCompletedEvent.getMetadata().getPreparedQuery().isPresent());
    assertEquals(queryCompletedEvent.getMetadata().getPreparedQuery().get(), selectQuery);
    assertEquals(queryCompletedEvent.getStatistics().getCompletedSplits(), SPLITS_PER_NODE + 2);
}
Also used : QueryCreatedEvent(com.facebook.presto.spi.eventlistener.QueryCreatedEvent) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) Session(com.facebook.presto.Session) Test(org.testng.annotations.Test)

Example 4 with QueryCompletedEvent

use of com.facebook.presto.spi.eventlistener.QueryCompletedEvent in project presto by prestodb.

the class TestEventListener method testOutputStats.

@Test
public void testOutputStats() throws Exception {
    // We expect the following events
    // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split
    int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1;
    MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", expectedEvents);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    QueryStats queryStats = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(result.getRowCount(), queryStats.getOutputPositions());
    assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows());
    runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryStats = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(1L, queryStats.getOutputPositions());
    assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows());
}
Also used : QueryCreatedEvent(com.facebook.presto.spi.eventlistener.QueryCreatedEvent) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) QueryId(com.facebook.presto.spi.QueryId) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Test(org.testng.annotations.Test)

Example 5 with QueryCompletedEvent

use of com.facebook.presto.spi.eventlistener.QueryCompletedEvent in project presto by prestodb.

the class TestEventListenerWithExchangeMaterialization method testRuntimeOptimizedStagesCorrectness.

@Test
public void testRuntimeOptimizedStagesCorrectness() throws Exception {
    // We expect one runtime optimized stage: 1.
    int expectedEvents = 2;
    QueryId queryId = runQueryAndWaitForEvents("SELECT phone, regionkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey", expectedEvents);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    QueryMetadata queryMetadata = queryCompletedEvent.getMetadata();
    Optional<List<StageId>> runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getQueryId(), queryId.toString());
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
    // Now, the following query should not trigger runtime optimizations, so should have empty list of runtime optimized stages.
    runQueryAndWaitForEvents("SELECT phone, regionkey FROM supplier INNER JOIN nation ON supplier.nationkey=nation.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertTrue(queryCompletedEvent.getMetadata().getRuntimeOptimizedStages().isEmpty());
    assertFalse(runtimeOptimizedStages.isPresent());
    // Now, the following query should have two optimized joins in a single stage (both on the same nationkey), therefore expect only one optimized stage: 1.
    runQueryAndWaitForEvents("SELECT supplier.phone, regionkey, custkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey INNER JOIN customer ON nation.nationkey=customer.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryMetadata = queryCompletedEvent.getMetadata();
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
    // Now, the following query should have two runtime optimized stages: 1 and 4, corresponding to the two join operations (on regionkey and nationkey respectively).
    runQueryAndWaitForEvents("WITH natreg AS (SELECT nation.regionkey, nationkey, region.name FROM region INNER JOIN nation ON nation.regionkey=region.regionkey) SELECT phone, regionkey FROM natreg INNER JOIN supplier ON supplier.nationkey=natreg.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryMetadata = queryCompletedEvent.getMetadata();
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 2);
    assertEquals(ImmutableSet.copyOf(queryMetadata.getRuntimeOptimizedStages()), ImmutableSet.of("1", "4"));
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 2);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
}
Also used : QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) QueryCreatedEvent(com.facebook.presto.spi.eventlistener.QueryCreatedEvent) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) QueryId(com.facebook.presto.spi.QueryId) ImmutableList(com.google.common.collect.ImmutableList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) Test(org.testng.annotations.Test)

Aggregations

QueryCompletedEvent (com.facebook.presto.spi.eventlistener.QueryCompletedEvent)12 QueryCreatedEvent (com.facebook.presto.spi.eventlistener.QueryCreatedEvent)6 QueryMetadata (com.facebook.presto.spi.eventlistener.QueryMetadata)6 Test (org.testng.annotations.Test)6 ImmutableList (com.google.common.collect.ImmutableList)4 QueryId (com.facebook.presto.spi.QueryId)3 SplitCompletedEvent (com.facebook.presto.spi.eventlistener.SplitCompletedEvent)3 List (java.util.List)3 Session (com.facebook.presto.Session)2 QueryStats (com.facebook.presto.execution.QueryStats)2 QueryContext (com.facebook.presto.spi.eventlistener.QueryContext)2 QueryFailureInfo (com.facebook.presto.spi.eventlistener.QueryFailureInfo)2 QueryIOMetadata (com.facebook.presto.spi.eventlistener.QueryIOMetadata)2 QueryStatistics (com.facebook.presto.spi.eventlistener.QueryStatistics)2 MaterializedResult (com.facebook.presto.testing.MaterializedResult)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 FailureInfo (com.facebook.presto.client.FailureInfo)1 NodeVersion (com.facebook.presto.client.NodeVersion)1 RuntimeStats (com.facebook.presto.common.RuntimeStats)1