Examples with QueryStats io.prestosql.execution.QueryStats used on opensource projects

Example 6 with QueryStats

use of io.prestosql.execution.QueryStats in project hetu-core by openlookeng.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        StageInfo outputStage = queryInfo.isRunningAsync() ? null : queryInfo.getOutputStage().orElse(null);
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime().toMillis();
        long logicalPlanning = queryStats.getTotalLogicalPlanningTime().toMillis();
        long distributedPlanning = queryStats.getDistributedPlanningTime().toMillis();
        long physicalPlanning = queryStats.getAnalysisTime().toMillis() - logicalPlanning;
        long syntaxAnalysisTime = queryStats.getTotalSyntaxAnalysisTime().toMillis();
        // Time spent waiting for required no. of worker nodes to be present
        long waiting = queryStats.getResourceWaitingTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        long firstTaskStartTime = queryEndTime.getMillis();
        long firstStageFirstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstStageFirstTaskStartTime = Math.min(firstStartTime.getMillis(), firstStageFirstTaskStartTime);
                }
                // only consider leaf stages for other stats.
                if (!stage.getSubStages().isEmpty()) {
                    continue;
                }
                firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0);
        // scheduling time is starting from end of plan time to  start of first task execution corresponding to any first stage.
        long scheduling = max(firstStageFirstTaskStartTime - queryStartTime.getMillis() - planning, 0);
        // executionInitializationTime is starting from first task of first stage to first task of leaf stage.
        long executionInitializationTime = max(firstTaskStartTime - firstStageFirstTaskStartTime, 0);
        long running = max(lastTaskEndTime - firstTaskStartTime, 0);
        long finishing = max(queryEndTime.getMillis() - lastTaskEndTime, 0);
        int spilledNodes = globalUniqueNodes(outputStage, true).size();
        long spilledWriteTimeMillisPerNode = spilledNodes > 0 ? (queryStats.getSpilledWriteTime().toMillis() / spilledNodes) : 0;
        long spilledReadTimeMillisPerNode = spilledNodes > 0 ? (queryStats.getSpilledReadTime().toMillis() / spilledNodes) : 0;
        logQueryTimeline(queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, syntaxAnalysisTime, planning, logicalPlanning, physicalPlanning, distributedPlanning, waiting, scheduling, executionInitializationTime, running, finishing, queryStartTime, queryEndTime, spilledWriteTimeMillisPerNode, spilledReadTimeMillisPerNode);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 7 with QueryStats

use of io.prestosql.execution.QueryStats in project hetu-core by openlookeng.

the class QuerySystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(QUERY_TABLE);
    List<QueryInfo> queryInfos = queryManager.getQueries().stream().map(BasicQueryInfo::getQueryId).map(queryId -> {
        try {
            return queryManager.getFullQueryInfo(queryId);
        } catch (NoSuchElementException e) {
            return null;
        }
    }).filter(Objects::nonNull).collect(toImmutableList());
    for (QueryInfo queryInfo : queryInfos) {
        QueryStats queryStats = queryInfo.getQueryStats();
        table.addRow(queryInfo.getQueryId().toString(), queryInfo.getState().toString(), queryInfo.getSession().getUser(), queryInfo.getSession().getSource().orElse(null), queryInfo.getQuery(), queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toMillis(queryStats.getDistributedPlanningTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
    }
    return table.build().cursor();
}

Example 8 with QueryStats

use of io.prestosql.execution.QueryStats in project hetu-core by openlookeng.

the class Execution method doExecute.

private Job doExecute() throws ExecutionFailureException {
    final String userQuery = QUERY_SPLITTER.splitToList(getJob().getQuery()).get(0);
    final JobOutputBuilder outputBuilder;
    job.setQueryStats(createNoOpQueryStats());
    try {
        outputBuilder = outputBuilderFactory.forJob(job);
    } catch (IOException e) {
        throw new ExecutionFailureException(job, "Could not create output builder for job", e);
    } catch (InvalidQueryException e) {
        throw new ExecutionFailureException(job, e.getMessage(), e);
    }
    final Persistor persistor = persistorFactory.getPersistor(job, job.getOutput());
    final String query = job.getOutput().processQuery(userQuery);
    if (!persistor.canPersist(authorizer)) {
        throw new ExecutionFailureException(job, "Not authorized to create tables", null);
    }
    final Set<Table> tables = new HashSet<>();
    try {
        tables.addAll(authorizer.tablesUsedByQuery(query));
    } catch (ParsingException e) {
        job.setError(new QueryError(e.getMessage(), null, -1, null, Optional.empty(), null, new ErrorLocation(e.getLineNumber(), e.getColumnNumber()), null));
        throw new ExecutionFailureException(job, "Invalid query, could not parse", e);
    }
    if (!authorizer.isAuthorizedRead(tables)) {
        job.setQueryStats(createNoOpQueryStats());
        throw new ExecutionFailureException(job, "Cannot access tables", null);
    }
    JobSessionContext jobSessionContext = JobSessionContext.buildFromClient(queryRunner.getSession());
    job.setSessionContext(jobSessionContext);
    QueryClient queryClient = new QueryClient(queryRunner, timeout, query);
    try {
        queryClient.executeWith((client) -> {
            if (client == null) {
                return null;
            }
            QueryStatusInfo statusInfo = client.currentStatusInfo();
            QueryData data = client.currentData();
            List<Column> resultColumns = null;
            JobState jobState = null;
            QueryError queryError = null;
            QueryStats queryStats = null;
            if (isCancelled) {
                throw new ExecutionFailureException(job, "Query was cancelled", null);
            }
            if (statusInfo.getError() != null) {
                queryError = statusInfo.getError();
                jobState = JobState.FAILED;
            }
            if ((statusInfo.getInfoUri() != null) && (jobState != JobState.FAILED)) {
                BasicQueryInfo queryInfo = queryInfoClient.from(statusInfo.getInfoUri(), statusInfo.getId());
                if (queryInfo != null) {
                    queryStats = queryInfo.getQueryStats();
                }
            }
            if (statusInfo.getInfoUri() != null && job.getInfoUri() == null) {
                URI infoUri = statusInfo.getInfoUri();
                String path = infoUri.getPath();
                path = path.substring(path.indexOf("query.html"));
                infoUri = URI.create(path + "?" + infoUri.getQuery());
                job.setInfoUri(infoUri);
            }
            if (statusInfo.getStats() != null) {
                jobState = JobState.fromStatementState(statusInfo.getStats().getState());
            }
            try {
                if (statusInfo.getColumns() != null) {
                    resultColumns = statusInfo.getColumns();
                    outputBuilder.addColumns(resultColumns);
                }
                if (data.getData() != null) {
                    List<List<Object>> resultsData = ImmutableList.copyOf(data.getData());
                    for (List<Object> row : resultsData) {
                        outputBuilder.addRow(row);
                    }
                }
            } catch (FileTooLargeException e) {
                throw new ExecutionFailureException(job, "Output file exceeded maximum configured filesize", e);
            }
            rlUpdateJobInfo(tables, resultColumns, queryStats, jobState, queryError);
            return null;
        });
    } catch (QueryTimeOutException e) {
        throw new ExecutionFailureException(job, format("Query exceeded maximum execution time of %s minutes", Duration.millis(e.getElapsedMs()).getStandardMinutes()), e);
    }
    QueryStatusInfo finalResults = queryClient.finalResults();
    if (finalResults != null && finalResults.getInfoUri() != null) {
        BasicQueryInfo queryInfo = queryInfoClient.from(finalResults.getInfoUri(), finalResults.getId());
        if (queryInfo != null) {
            updateJobInfo(null, null, queryInfo.getQueryStats(), JobState.fromStatementState(finalResults.getStats().getState()), finalResults.getError());
        }
    }
    if (job.getState() != JobState.FAILED) {
        URI location = persistor.persist(outputBuilder, job);
        if (location != null) {
            job.getOutput().setLocation(location);
        }
    } else {
        throw new ExecutionFailureException(job, null, null);
    }
    return getJob();
}

Example 9 with QueryStats

use of io.prestosql.execution.QueryStats in project hetu-core by openlookeng.

the class Execution method createNoOpQueryStats.

public static QueryStats createNoOpQueryStats() {
    DateTime now = DateTime.now();
    io.airlift.units.Duration zeroDuration = new io.airlift.units.Duration(0, TimeUnit.SECONDS);
    DataSize zeroData = new DataSize(0, DataSize.Unit.BYTE);
    return new QueryStats(now, null, now, now, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, zeroDuration, 0, 0, 0, 0, 0, 0, 0, 0, 0.0, zeroData, zeroData, zeroData, zeroData, zeroData, zeroData, zeroData, zeroData, zeroData, false, zeroDuration, zeroDuration, zeroDuration, false, ImmutableSet.of(), zeroData, 0, zeroData, 0, zeroData, 0, zeroData, 0, zeroData, 0, zeroData, ImmutableList.of(), ImmutableList.of());
}

Example 10 with QueryStats

use of io.prestosql.execution.QueryStats in project hetu-core by openlookeng.

the class QueryResource method toFullQueryInfo.

private static QueryInfo toFullQueryInfo(DispatchQuery query) {
    checkArgument(query.isDone(), "query is not done");
    BasicQueryInfo info = query.getBasicQueryInfo();
    BasicQueryStats stats = info.getQueryStats();
    QueryStats queryStats = new QueryStats(query.getCreateTime(), query.getExecutionStartTime().orElse(null), query.getLastHeartbeat(), query.getEndTime().orElse(null), stats.getElapsedTime(), stats.getQueuedTime(), ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, 0, 0, 0, 0, 0, 0, 0, 0, 0, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, ZERO_BYTES, info.isScheduled(), ZERO_MILLIS, ZERO_MILLIS, ZERO_MILLIS, false, ImmutableSet.of(), ZERO_BYTES, 0, ZERO_BYTES, 0, ZERO_BYTES, 0, ZERO_BYTES, 0, ZERO_BYTES, 0, ZERO_BYTES, ImmutableList.of(), ImmutableList.of());
    return new QueryInfo(info.getQueryId(), info.getSession(), info.getState(), info.getMemoryPool(), info.isScheduled(), info.getSelf(), ImmutableList.of(), info.getQuery(), info.getPreparedQuery(), queryStats, Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, null, Optional.empty(), query.getDispatchInfo().getFailureInfo().orElse(null), info.getErrorCode(), ImmutableList.of(), ImmutableSet.of(), Optional.empty(), true, info.getResourceGroupId(), false);
}