Examples with QueryStats com.facebook.presto.execution.QueryStats used on opensource projects

Example 1 with QueryStats

use of com.facebook.presto.execution.QueryStats in project presto by prestodb.

the class QueryMonitor method queryCompletedEvent.

public void queryCompletedEvent(QueryInfo queryInfo) {
    try {
        Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
        if (queryInfo.getFailureInfo() != null) {
            FailureInfo failureInfo = queryInfo.getFailureInfo();
            Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
            queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
        }
        ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
        for (Input input : queryInfo.getInputs()) {
            inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
        }
        QueryStats queryStats = queryInfo.getQueryStats();
        Optional<QueryOutputMetadata> output = Optional.empty();
        if (queryInfo.getOutput().isPresent()) {
            Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
            output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
        }
        eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
        logQueryTimeline(queryInfo);
    } catch (JsonProcessingException e) {
        throw Throwables.propagate(e);
    }
}

Example 2 with QueryStats

use of com.facebook.presto.execution.QueryStats in project presto by prestodb.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime() == null ? 0 : queryStats.getTotalPlanningTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        // long lastSchedulingCompletion = 0;
        long firstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            // only consider leaf stages
            if (!stage.getSubStages().isEmpty()) {
                continue;
            }
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = queryEndTime.getMillis() - queryStartTime.getMillis();
        long scheduling = firstTaskStartTime - queryStartTime.getMillis() - planning;
        long running = lastTaskEndTime - firstTaskStartTime;
        long finishing = queryEndTime.getMillis() - lastTaskEndTime;
        log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), max(elapsed, 0), max(planning, 0), max(scheduling, 0), max(running, 0), max(finishing, 0), queryStartTime, queryEndTime);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 3 with QueryStats

use of com.facebook.presto.execution.QueryStats in project presto by prestodb.

the class PrestoSparkQueryExecutionFactory method createQueryInfo.

private static QueryInfo createQueryInfo(Session session, String query, QueryState queryState, Optional<PlanAndMore> planAndMore, Optional<String> sparkQueueName, Optional<ExecutionFailureInfo> failureInfo, QueryStateTimer queryStateTimer, Optional<StageInfo> rootStage, WarningCollector warningCollector) {
    checkArgument(failureInfo.isPresent() || queryState != FAILED, "unexpected query state: %s", queryState);
    int peakRunningTasks = 0;
    long peakUserMemoryReservationInBytes = 0;
    long peakTotalMemoryReservationInBytes = 0;
    long peakTaskUserMemoryInBytes = 0;
    long peakTaskTotalMemoryInBytes = 0;
    long peakNodeTotalMemoryInBytes = 0;
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageExecutionInfo stageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo();
        for (TaskInfo taskInfo : stageExecutionInfo.getTasks()) {
            // there's no way to know how many tasks were running in parallel in Spark
            // for now let's assume that all the tasks were running in parallel
            peakRunningTasks++;
            long taskPeakUserMemoryInBytes = taskInfo.getStats().getPeakUserMemoryInBytes();
            long taskPeakTotalMemoryInBytes = taskInfo.getStats().getPeakTotalMemoryInBytes();
            peakUserMemoryReservationInBytes += taskPeakUserMemoryInBytes;
            peakTotalMemoryReservationInBytes += taskPeakTotalMemoryInBytes;
            peakTaskUserMemoryInBytes = max(peakTaskUserMemoryInBytes, taskPeakUserMemoryInBytes);
            peakTaskTotalMemoryInBytes = max(peakTaskTotalMemoryInBytes, taskPeakTotalMemoryInBytes);
            peakNodeTotalMemoryInBytes = max(taskInfo.getStats().getPeakNodeTotalMemoryInBytes(), peakNodeTotalMemoryInBytes);
        }
    }
    QueryStats queryStats = QueryStats.create(queryStateTimer, rootStage, peakRunningTasks, succinctBytes(peakUserMemoryReservationInBytes), succinctBytes(peakTotalMemoryReservationInBytes), succinctBytes(peakTaskUserMemoryInBytes), succinctBytes(peakTaskTotalMemoryInBytes), succinctBytes(peakNodeTotalMemoryInBytes), session.getRuntimeStats());
    return new QueryInfo(session.getQueryId(), session.toSessionRepresentation(), queryState, new MemoryPoolId("spark-memory-pool"), queryStats.isScheduled(), URI.create("http://fake.invalid/query/" + session.getQueryId()), planAndMore.map(PlanAndMore::getFieldNames).orElse(ImmutableList.of()), query, Optional.empty(), Optional.empty(), queryStats, Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, planAndMore.flatMap(PlanAndMore::getUpdateType).orElse(null), rootStage, failureInfo.orElse(null), failureInfo.map(ExecutionFailureInfo::getErrorCode).orElse(null), warningCollector.getWarnings(), planAndMore.map(PlanAndMore::getInputs).orElse(ImmutableSet.of()), planAndMore.flatMap(PlanAndMore::getOutput), true, sparkQueueName.map(ResourceGroupId::new), planAndMore.flatMap(PlanAndMore::getQueryType), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of());
}

Example 4 with QueryStats

use of com.facebook.presto.execution.QueryStats in project presto by prestodb.

the class TestQueryStateInfo method testQueryStateInfoCreation.

@Test
public void testQueryStateInfoCreation() {
    QueryInfo queryInfo = createQueryInfo("query_id_test", RUNNING, "SELECT * FROM foo");
    QueryStateInfo queryStateInfo = createQueryStateInfo(new BasicQueryInfo(queryInfo));
    assertEquals(queryStateInfo.getQueryId(), queryInfo.getQueryId());
    assertEquals(queryStateInfo.getQueryState(), queryInfo.getState());
    assertEquals(queryStateInfo.getResourceGroupId(), queryInfo.getResourceGroupId());
    assertEquals(queryStateInfo.getQuery(), queryInfo.getQuery());
    assertEquals(queryStateInfo.getCreateTime(), queryInfo.getQueryStats().getCreateTime());
    assertEquals(queryStateInfo.getUser(), queryInfo.getSession().getUser());
    assertEquals(queryStateInfo.isAuthenticated(), queryInfo.getSession().getPrincipal().isPresent());
    assertEquals(queryStateInfo.getSource(), queryInfo.getSession().getSource());
    assertEquals(queryStateInfo.getClientInfo(), queryInfo.getSession().getClientInfo());
    assertEquals(queryStateInfo.getCatalog(), queryInfo.getSession().getCatalog());
    assertEquals(queryStateInfo.getSchema(), queryInfo.getSession().getSchema());
    assertEquals(queryStateInfo.getWarningCodes(), ImmutableList.of("WARNING_123"));
    assertTrue(queryStateInfo.getProgress().isPresent());
    assertEquals(queryStateInfo.getErrorCode(), Optional.ofNullable(queryInfo.getErrorCode()));
    QueryProgressStats progress = queryStateInfo.getProgress().get();
    QueryStats stats = queryInfo.getQueryStats();
    assertEquals(progress.getElapsedTimeMillis(), stats.getElapsedTime().toMillis());
    assertEquals(progress.getQueuedTimeMillis(), stats.getQueuedTime().toMillis());
    assertEquals(progress.getExecutionTimeMillis(), stats.getExecutionTime().toMillis());
    assertEquals(progress.getCpuTimeMillis(), stats.getTotalCpuTime().toMillis());
    assertEquals(progress.getScheduledTimeMillis(), stats.getTotalScheduledTime().toMillis());
    assertEquals(progress.getCurrentMemoryBytes(), stats.getUserMemoryReservation().toBytes());
    assertEquals(progress.getPeakMemoryBytes(), stats.getPeakUserMemoryReservation().toBytes());
    assertEquals(progress.getPeakTotalMemoryBytes(), stats.getPeakTotalMemoryReservation().toBytes());
    assertEquals(progress.getCumulativeUserMemory(), stats.getCumulativeUserMemory());
    assertEquals(progress.getCumulativeTotalMemory(), stats.getCumulativeTotalMemory());
    assertEquals(progress.getInputRows(), stats.getRawInputPositions());
    assertEquals(progress.getInputBytes(), stats.getRawInputDataSize().toBytes());
    assertEquals(progress.isBlocked(), stats.isFullyBlocked());
    assertEquals(progress.getBlockedReasons(), Optional.of(stats.getBlockedReasons()));
    assertEquals(progress.getProgressPercentage(), stats.getProgressPercentage());
    assertEquals(progress.getQueuedDrivers(), stats.getQueuedDrivers());
    assertEquals(progress.getRunningDrivers(), stats.getRunningDrivers());
    assertEquals(progress.getCompletedDrivers(), stats.getCompletedDrivers());
}

Example 5 with QueryStats

use of com.facebook.presto.execution.QueryStats in project presto by prestodb.

the class QuerySystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(QUERY_TABLE);
    List<QueryInfo> queryInfos = queryManager.getQueries().stream().map(BasicQueryInfo::getQueryId).map(queryId -> {
        try {
            return queryManager.getFullQueryInfo(queryId);
        } catch (NoSuchElementException e) {
            return null;
        }
    }).filter(Objects::nonNull).collect(toImmutableList());
    for (QueryInfo queryInfo : queryInfos) {
        QueryStats queryStats = queryInfo.getQueryStats();
        table.addRow(queryInfo.getQueryId().toString(), queryInfo.getState().toString(), queryInfo.getSession().getUser(), queryInfo.getSession().getSource().orElse(null), queryInfo.getQuery(), queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
    }
    return table.build().cursor();
}