Examples with TaskStats com.facebook.presto.operator.TaskStats used on opensource projects

Example 1 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(List<StageInfo> stages, Metadata metadata, Session session) {
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = stages.stream().flatMap(stage -> getAllStages(Optional.of(stage)).stream()).collect(toImmutableList());
    for (StageInfo stageInfo : allStages) {
        Map<PlanNodeId, PlanNodeStats> aggregatedStats = new HashMap<>();
        List<PlanNodeStats> planNodeStats = stageInfo.getTasks().stream().map(TaskInfo::getStats).flatMap(taskStats -> getPlanNodeStats(taskStats).stream()).collect(toList());
        for (PlanNodeStats stats : planNodeStats) {
            aggregatedStats.merge(stats.getPlanNodeId(), stats, PlanNodeStats::merge);
        }
        builder.append(formatFragment(metadata, session, stageInfo.getPlan(), Optional.of(stageInfo.getStageStats()), Optional.of(aggregatedStats)));
    }
    return builder.toString();
}

Example 2 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime() == null ? 0 : queryStats.getTotalPlanningTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        // long lastSchedulingCompletion = 0;
        long firstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            // only consider leaf stages
            if (!stage.getSubStages().isEmpty()) {
                continue;
            }
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = queryEndTime.getMillis() - queryStartTime.getMillis();
        long scheduling = firstTaskStartTime - queryStartTime.getMillis() - planning;
        long running = lastTaskEndTime - firstTaskStartTime;
        long finishing = queryEndTime.getMillis() - lastTaskEndTime;
        log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), max(elapsed, 0), max(planning, 0), max(scheduling, 0), max(running, 0), max(finishing, 0), queryStartTime, queryEndTime);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 3 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method createTaskStatus.

private TaskStatus createTaskStatus(TaskHolder taskHolder) {
    long taskStatusAgeInMilis = System.currentTimeMillis() - creationTimeInMillis;
    // Always return a new TaskInfo with a larger version number;
    // otherwise a client will not accept the update
    long versionNumber = nextTaskInfoVersion.getAndIncrement();
    TaskState state = taskStateMachine.getState();
    List<ExecutionFailureInfo> failures = ImmutableList.of();
    if (state == FAILED) {
        failures = toFailures(taskStateMachine.getFailureCauses());
    }
    int queuedPartitionedDrivers = 0;
    long queuedPartitionedSplitsWeight = 0L;
    int runningPartitionedDrivers = 0;
    long runningPartitionedSplitsWeight = 0L;
    long physicalWrittenDataSizeInBytes = 0L;
    long userMemoryReservationInBytes = 0L;
    long systemMemoryReservationInBytes = 0L;
    // TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
    Set<Lifespan> completedDriverGroups = ImmutableSet.of();
    long fullGcCount = 0;
    long fullGcTimeInMillis = 0L;
    long totalCpuTimeInNanos = 0L;
    if (taskHolder.getFinalTaskInfo() != null) {
        TaskStats taskStats = taskHolder.getFinalTaskInfo().getStats();
        queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
        queuedPartitionedSplitsWeight = taskStats.getQueuedPartitionedSplitsWeight();
        runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
        runningPartitionedSplitsWeight = taskStats.getRunningPartitionedSplitsWeight();
        physicalWrittenDataSizeInBytes = taskStats.getPhysicalWrittenDataSizeInBytes();
        userMemoryReservationInBytes = taskStats.getUserMemoryReservationInBytes();
        systemMemoryReservationInBytes = taskStats.getSystemMemoryReservationInBytes();
        fullGcCount = taskStats.getFullGcCount();
        fullGcTimeInMillis = taskStats.getFullGcTimeInMillis();
        totalCpuTimeInNanos = taskStats.getTotalCpuTimeInNanos();
    } else if (taskHolder.getTaskExecution() != null) {
        long physicalWrittenBytes = 0;
        TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
        for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
            PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
            queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
            queuedPartitionedSplitsWeight += pipelineStatus.getQueuedPartitionedSplitsWeight();
            runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
            runningPartitionedSplitsWeight += pipelineStatus.getRunningPartitionedSplitsWeight();
            physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
            totalCpuTimeInNanos += pipelineContext.getPipelineStats().getTotalCpuTimeInNanos();
        }
        physicalWrittenDataSizeInBytes = physicalWrittenBytes;
        userMemoryReservationInBytes = taskContext.getMemoryReservation().toBytes();
        systemMemoryReservationInBytes = taskContext.getSystemMemoryReservation().toBytes();
        completedDriverGroups = taskContext.getCompletedDriverGroups();
        fullGcCount = taskContext.getFullGcCount();
        fullGcTimeInMillis = taskContext.getFullGcTime().toMillis();
    }
    return new TaskStatus(taskInstanceId.getUuidLeastSignificantBits(), taskInstanceId.getUuidMostSignificantBits(), versionNumber, state, location, completedDriverGroups, failures, queuedPartitionedDrivers, runningPartitionedDrivers, outputBuffer.getUtilization(), isOutputBufferOverutilized(), physicalWrittenDataSizeInBytes, userMemoryReservationInBytes, systemMemoryReservationInBytes, queryContext.getPeakNodeTotalMemory(), fullGcCount, fullGcTimeInMillis, totalCpuTimeInNanos, taskStatusAgeInMilis, queuedPartitionedSplitsWeight, runningPartitionedSplitsWeight);
}

Example 4 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method createTaskInfo.

private TaskInfo createTaskInfo(TaskHolder taskHolder) {
    TaskStats taskStats = getTaskStats(taskHolder);
    Set<PlanNodeId> noMoreSplits = getNoMoreSplits(taskHolder);
    MetadataUpdates metadataRequests = getMetadataUpdateRequests(taskHolder);
    TaskStatus taskStatus = createTaskStatus(taskHolder);
    return new TaskInfo(taskStateMachine.getTaskId(), taskStatus, lastHeartbeat.get(), outputBuffer.getInfo(), noMoreSplits, taskStats, needsPlan.get(), metadataRequests, nodeId);
}

Example 5 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method getTaskStats.

private TaskStats getTaskStats(TaskHolder taskHolder) {
    TaskInfo finalTaskInfo = taskHolder.getFinalTaskInfo();
    if (finalTaskInfo != null) {
        return finalTaskInfo.getStats();
    }
    SqlTaskExecution taskExecution = taskHolder.getTaskExecution();
    if (taskExecution != null) {
        return taskExecution.getTaskContext().getTaskStats();
    }
    // if the task completed without creation, set end time
    DateTime endTime = taskStateMachine.getState().isDone() ? DateTime.now() : null;
    return new TaskStats(taskStateMachine.getCreatedTime(), endTime);
}