Examples with StageInfo io.prestosql.execution.StageInfo used on opensource projects

Example 6 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        StageInfo outputStage = queryInfo.isRunningAsync() ? null : queryInfo.getOutputStage().orElse(null);
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime().toMillis();
        long logicalPlanning = queryStats.getTotalLogicalPlanningTime().toMillis();
        long distributedPlanning = queryStats.getDistributedPlanningTime().toMillis();
        long physicalPlanning = queryStats.getAnalysisTime().toMillis() - logicalPlanning;
        long syntaxAnalysisTime = queryStats.getTotalSyntaxAnalysisTime().toMillis();
        // Time spent waiting for required no. of worker nodes to be present
        long waiting = queryStats.getResourceWaitingTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        long firstTaskStartTime = queryEndTime.getMillis();
        long firstStageFirstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstStageFirstTaskStartTime = Math.min(firstStartTime.getMillis(), firstStageFirstTaskStartTime);
                }
                // only consider leaf stages for other stats.
                if (!stage.getSubStages().isEmpty()) {
                    continue;
                }
                firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0);
        // scheduling time is starting from end of plan time to  start of first task execution corresponding to any first stage.
        long scheduling = max(firstStageFirstTaskStartTime - queryStartTime.getMillis() - planning, 0);
        // executionInitializationTime is starting from first task of first stage to first task of leaf stage.
        long executionInitializationTime = max(firstTaskStartTime - firstStageFirstTaskStartTime, 0);
        long running = max(lastTaskEndTime - firstTaskStartTime, 0);
        long finishing = max(queryEndTime.getMillis() - lastTaskEndTime, 0);
        int spilledNodes = globalUniqueNodes(outputStage, true).size();
        long spilledWriteTimeMillisPerNode = spilledNodes > 0 ? (queryStats.getSpilledWriteTime().toMillis() / spilledNodes) : 0;
        long spilledReadTimeMillisPerNode = spilledNodes > 0 ? (queryStats.getSpilledReadTime().toMillis() / spilledNodes) : 0;
        logQueryTimeline(queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, syntaxAnalysisTime, planning, logicalPlanning, physicalPlanning, distributedPlanning, waiting, scheduling, executionInitializationTime, running, finishing, queryStartTime, queryEndTime, spilledWriteTimeMillisPerNode, spilledReadTimeMillisPerNode);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 7 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class SqlQueryScheduler method buildStageInfo.

private StageInfo buildStageInfo(StageId stageId, Map<StageId, StageInfo> stageInfos) {
    StageInfo parent = stageInfos.get(stageId);
    checkArgument(parent != null, "No stageInfo for %s", parent);
    List<StageInfo> childStages = stageLinkages.get(stageId).getChildStageIds().stream().map(childStageId -> buildStageInfo(childStageId, stageInfos)).collect(toImmutableList());
    if (childStages.isEmpty()) {
        return parent;
    }
    return new StageInfo(parent.getStageId(), parent.getState(), parent.getSelf(), parent.getPlan(), parent.getTypes(), parent.getStageStats(), parent.getTasks(), childStages, parent.getTables(), parent.getFailureCause());
}

Example 8 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class StatsAndCosts method reconstructStatsAndCosts.

private static void reconstructStatsAndCosts(StageInfo stage, ImmutableMap.Builder<PlanNodeId, PlanNodeStatsEstimate> planNodeStats, ImmutableMap.Builder<PlanNodeId, PlanCostEstimate> planNodeCosts, Set<PlanNodeId> visitedPlanNodeId) {
    PlanFragment planFragment = stage.getPlan();
    if (planFragment != null) {
        if (!visitedPlanNodeId.contains(planFragment.getRoot().getId())) {
            visitedPlanNodeId.add(planFragment.getRoot().getId());
            planNodeStats.putAll(planFragment.getStatsAndCosts().getStats());
            planNodeCosts.putAll(planFragment.getStatsAndCosts().getCosts());
        }
    }
    for (StageInfo subStage : stage.getSubStages()) {
        reconstructStatsAndCosts(subStage, planNodeStats, planNodeCosts, visitedPlanNodeId);
    }
}

Example 9 with StageInfo

use of io.prestosql.execution.StageInfo in project hetu-core by openlookeng.

the class Query method toStageStats.

private static StageStats toStageStats(StageInfo stageInfo) {
    if (stageInfo == null) {
        return null;
    }
    io.prestosql.execution.StageStats stageStats = stageInfo.getStageStats();
    ImmutableList.Builder<StageStats> subStages = ImmutableList.builder();
    for (StageInfo subStage : stageInfo.getSubStages()) {
        subStages.add(toStageStats(subStage));
    }
    Set<String> uniqueNodes = new HashSet<>();
    for (TaskInfo task : stageInfo.getTasks()) {
        // todo add nodeId to TaskInfo
        URI uri = task.getTaskStatus().getSelf();
        uniqueNodes.add(uri.getHost() + ":" + uri.getPort());
    }
    return StageStats.builder().setStageId(String.valueOf(stageInfo.getStageId().getId())).setState(stageInfo.getState().toString()).setDone(stageInfo.getState().isDone()).setNodes(uniqueNodes.size()).setTotalSplits(stageStats.getTotalDrivers()).setQueuedSplits(stageStats.getQueuedDrivers()).setRunningSplits(stageStats.getRunningDrivers() + stageStats.getBlockedDrivers()).setCompletedSplits(stageStats.getCompletedDrivers()).setCpuTimeMillis(stageStats.getTotalCpuTime().toMillis()).setWallTimeMillis(stageStats.getTotalScheduledTime().toMillis()).setProcessedRows(stageStats.getRawInputPositions()).setProcessedBytes(stageStats.getRawInputDataSize().toBytes()).setSubStages(subStages.build()).build();
}