Search in sources :

Example 1 with QueryIOMetadata

use of com.facebook.presto.spi.eventlistener.QueryIOMetadata in project presto by prestodb.

the class QueryMonitor method queryCompletedEvent.

public void queryCompletedEvent(QueryInfo queryInfo) {
    try {
        Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
        if (queryInfo.getFailureInfo() != null) {
            FailureInfo failureInfo = queryInfo.getFailureInfo();
            Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
            queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
        }
        ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
        for (Input input : queryInfo.getInputs()) {
            inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
        }
        QueryStats queryStats = queryInfo.getQueryStats();
        Optional<QueryOutputMetadata> output = Optional.empty();
        if (queryInfo.getOutput().isPresent()) {
            Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
            output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
        }
        eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
        logQueryTimeline(queryInfo);
    } catch (JsonProcessingException e) {
        throw Throwables.propagate(e);
    }
}
Also used : TaskStats(com.facebook.presto.operator.TaskStats) QueryContext(com.facebook.presto.spi.eventlistener.QueryContext) NodeInfo(io.airlift.node.NodeInfo) TaskState(com.facebook.presto.execution.TaskState) NodeVersion(com.facebook.presto.client.NodeVersion) TableFinishInfo(com.facebook.presto.operator.TableFinishInfo) EventListenerManager(com.facebook.presto.eventlistener.EventListenerManager) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) OperatorStats(com.facebook.presto.operator.OperatorStats) StageInfo(com.facebook.presto.execution.StageInfo) Instant.ofEpochMilli(java.time.Instant.ofEpochMilli) Duration(java.time.Duration) Map(java.util.Map) QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) TransactionId(com.facebook.presto.transaction.TransactionId) SplitStatistics(com.facebook.presto.spi.eventlistener.SplitStatistics) Column(com.facebook.presto.execution.Column) ImmutableMap(com.google.common.collect.ImmutableMap) SplitCompletedEvent(com.facebook.presto.spi.eventlistener.SplitCompletedEvent) Collectors(java.util.stream.Collectors) List(java.util.List) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) Input(com.facebook.presto.execution.Input) QueryInfo(com.facebook.presto.execution.QueryInfo) Optional(java.util.Optional) Math.max(java.lang.Math.max) ConnectorId(com.facebook.presto.connector.ConnectorId) Duration.ofMillis(java.time.Duration.ofMillis) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) JsonCodec(io.airlift.json.JsonCodec) DriverStats(com.facebook.presto.operator.DriverStats) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) Logger(io.airlift.log.Logger) Inject(javax.inject.Inject) ImmutableList(com.google.common.collect.ImmutableList) FailureInfo(com.facebook.presto.client.FailureInfo) Objects.requireNonNull(java.util.Objects.requireNonNull) Math.toIntExact(java.lang.Math.toIntExact) Nullable(javax.annotation.Nullable) QueryOutputMetadata(com.facebook.presto.spi.eventlistener.QueryOutputMetadata) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) Throwables(com.google.common.base.Throwables) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) QueryInputMetadata(com.facebook.presto.spi.eventlistener.QueryInputMetadata) QueryStats(com.facebook.presto.execution.QueryStats) SplitFailureInfo(com.facebook.presto.spi.eventlistener.SplitFailureInfo) TaskId(com.facebook.presto.execution.TaskId) QueryCreatedEvent(com.facebook.presto.spi.eventlistener.QueryCreatedEvent) TaskInfo(com.facebook.presto.execution.TaskInfo) TableFinishInfo(com.facebook.presto.operator.TableFinishInfo) QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) ImmutableList(com.google.common.collect.ImmutableList) QueryOutputMetadata(com.facebook.presto.spi.eventlistener.QueryOutputMetadata) QueryContext(com.facebook.presto.spi.eventlistener.QueryContext) QueryInputMetadata(com.facebook.presto.spi.eventlistener.QueryInputMetadata) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) TaskInfo(com.facebook.presto.execution.TaskInfo) Input(com.facebook.presto.execution.Input) QueryStats(com.facebook.presto.execution.QueryStats) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) QueryFailureInfo(com.facebook.presto.spi.eventlistener.QueryFailureInfo) FailureInfo(com.facebook.presto.client.FailureInfo) SplitFailureInfo(com.facebook.presto.spi.eventlistener.SplitFailureInfo) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException)

Example 2 with QueryIOMetadata

use of com.facebook.presto.spi.eventlistener.QueryIOMetadata in project presto-audit by yahoojapan.

the class TestHelper method setUp.

public void setUp() {
    try {
        uri = new URI("http://example.com:8080/v1/query/20170521_140224_00002_gd5k3");
        statistics = new QueryStatistics(Duration.ofMillis(100), Duration.ofMillis(200), Duration.ofMillis(300), Optional.of(Duration.ofMillis(400)), Optional.of(Duration.ofMillis(500)), 10001, 10002, 10003, 14, 0, 0, 0, 2048.0, 4096, true, new ArrayList<StageCpuDistribution>() {
        }, new ArrayList<String>() {

            {
                add("operatorSummaries 01");
            }
        });
        context = new QueryContext("test-user", Optional.of("principal"), Optional.of("example.com"), Optional.of("StatementClient 0.167"), Optional.of("clientInfo"), new HashSet<>(), Optional.of("presto-cli"), Optional.of("catalog"), Optional.of("schema"), Optional.of(""), new HashMap<>(), "127.0.0.1", "0.175", "environment");
        ioMetadata = new QueryIOMetadata(new ArrayList<QueryInputMetadata>(), Optional.empty());
        ZoneId jst_zone = ZoneId.of("Asia/Tokyo");
        createTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 0, 0, jst_zone).toInstant();
        executionStartTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 1, 0, jst_zone).toInstant();
        endTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 3, 0, jst_zone).toInstant();
    } catch (Exception ignoreException) {
    }
}
Also used : ZoneId(java.time.ZoneId) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) QueryContext(com.facebook.presto.spi.eventlistener.QueryContext) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) URI(java.net.URI) HashSet(java.util.HashSet)

Example 3 with QueryIOMetadata

use of com.facebook.presto.spi.eventlistener.QueryIOMetadata in project presto by prestodb.

the class QueryMonitor method getQueryIOMetadata.

private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) {
    ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
    for (Input input : queryInfo.getInputs()) {
        inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::getName).collect(Collectors.toList()), input.getConnectorInfo(), input.getStatistics()));
    }
    Optional<QueryOutputMetadata> output = Optional.empty();
    if (queryInfo.getOutput().isPresent()) {
        Optional<TableFinishInfo> tableFinishInfo = queryInfo.getQueryStats().getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
        output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getSerializedConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
    }
    return new QueryIOMetadata(inputs.build(), output);
}
Also used : Input(com.facebook.presto.execution.Input) TableFinishInfo(com.facebook.presto.operator.TableFinishInfo) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) QueryOutputMetadata(com.facebook.presto.spi.eventlistener.QueryOutputMetadata) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata) QueryInputMetadata(com.facebook.presto.spi.eventlistener.QueryInputMetadata)

Example 4 with QueryIOMetadata

use of com.facebook.presto.spi.eventlistener.QueryIOMetadata in project presto by prestodb.

the class QueryMonitor method queryImmediateFailureEvent.

public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailureInfo failure) {
    eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getPreparedQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), queryInfo.getSession().getTraceToken()), new QueryStatistics(ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(queryInfo.getQueryStats().getWaitingForPrerequisitesTime().toMillis()), ofMillis(queryInfo.getQueryStats().getQueuedTime().toMillis()), ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(0), ofMillis(0), Optional.empty(), ofMillis(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, true, new RuntimeStats()), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), new QueryIOMetadata(ImmutableList.of(), Optional.empty()), createQueryFailureInfo(failure, Optional.empty()), ImmutableList.of(), queryInfo.getQueryType(), ImmutableList.of(), ofEpochMilli(queryInfo.getQueryStats().getCreateTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis()), ofEpochMilli(queryInfo.getQueryStats().getEndTime().getMillis()), ImmutableList.of(), ImmutableList.of(), Optional.empty()));
    logQueryTimeline(queryInfo);
}
Also used : QueryMetadata(com.facebook.presto.spi.eventlistener.QueryMetadata) QueryCompletedEvent(com.facebook.presto.spi.eventlistener.QueryCompletedEvent) QueryStatistics(com.facebook.presto.spi.eventlistener.QueryStatistics) RuntimeStats(com.facebook.presto.common.RuntimeStats) QueryIOMetadata(com.facebook.presto.spi.eventlistener.QueryIOMetadata)

Aggregations

QueryIOMetadata (com.facebook.presto.spi.eventlistener.QueryIOMetadata)4 QueryStatistics (com.facebook.presto.spi.eventlistener.QueryStatistics)3 Input (com.facebook.presto.execution.Input)2 TableFinishInfo (com.facebook.presto.operator.TableFinishInfo)2 QueryCompletedEvent (com.facebook.presto.spi.eventlistener.QueryCompletedEvent)2 QueryContext (com.facebook.presto.spi.eventlistener.QueryContext)2 QueryInputMetadata (com.facebook.presto.spi.eventlistener.QueryInputMetadata)2 QueryMetadata (com.facebook.presto.spi.eventlistener.QueryMetadata)2 QueryOutputMetadata (com.facebook.presto.spi.eventlistener.QueryOutputMetadata)2 ImmutableList (com.google.common.collect.ImmutableList)2 FailureInfo (com.facebook.presto.client.FailureInfo)1 NodeVersion (com.facebook.presto.client.NodeVersion)1 RuntimeStats (com.facebook.presto.common.RuntimeStats)1 ConnectorId (com.facebook.presto.connector.ConnectorId)1 EventListenerManager (com.facebook.presto.eventlistener.EventListenerManager)1 Column (com.facebook.presto.execution.Column)1 QueryInfo (com.facebook.presto.execution.QueryInfo)1 QueryStats (com.facebook.presto.execution.QueryStats)1 StageInfo (com.facebook.presto.execution.StageInfo)1 TaskId (com.facebook.presto.execution.TaskId)1