use of com.facebook.presto.operator.TaskStats in project presto by prestodb.
the class PlanPrinter method textDistributedPlan.
public static String textDistributedPlan(List<StageInfo> stages, Metadata metadata, Session session) {
StringBuilder builder = new StringBuilder();
List<StageInfo> allStages = stages.stream().flatMap(stage -> getAllStages(Optional.of(stage)).stream()).collect(toImmutableList());
for (StageInfo stageInfo : allStages) {
Map<PlanNodeId, PlanNodeStats> aggregatedStats = new HashMap<>();
List<PlanNodeStats> planNodeStats = stageInfo.getTasks().stream().map(TaskInfo::getStats).flatMap(taskStats -> getPlanNodeStats(taskStats).stream()).collect(toList());
for (PlanNodeStats stats : planNodeStats) {
aggregatedStats.merge(stats.getPlanNodeId(), stats, PlanNodeStats::merge);
}
builder.append(formatFragment(metadata, session, stageInfo.getPlan(), Optional.of(stageInfo.getStageStats()), Optional.of(aggregatedStats)));
}
return builder.toString();
}
use of com.facebook.presto.operator.TaskStats in project presto by prestodb.
the class QueryMonitor method logQueryTimeline.
private static void logQueryTimeline(QueryInfo queryInfo) {
try {
QueryStats queryStats = queryInfo.getQueryStats();
DateTime queryStartTime = queryStats.getCreateTime();
DateTime queryEndTime = queryStats.getEndTime();
// query didn't finish cleanly
if (queryStartTime == null || queryEndTime == null) {
return;
}
// planning duration -- start to end of planning
long planning = queryStats.getTotalPlanningTime() == null ? 0 : queryStats.getTotalPlanningTime().toMillis();
List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
// long lastSchedulingCompletion = 0;
long firstTaskStartTime = queryEndTime.getMillis();
long lastTaskStartTime = queryStartTime.getMillis() + planning;
long lastTaskEndTime = queryStartTime.getMillis() + planning;
for (StageInfo stage : stages) {
// only consider leaf stages
if (!stage.getSubStages().isEmpty()) {
continue;
}
for (TaskInfo taskInfo : stage.getTasks()) {
TaskStats taskStats = taskInfo.getStats();
DateTime firstStartTime = taskStats.getFirstStartTime();
if (firstStartTime != null) {
firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
}
DateTime lastStartTime = taskStats.getLastStartTime();
if (lastStartTime != null) {
lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
}
DateTime endTime = taskStats.getEndTime();
if (endTime != null) {
lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
}
}
}
long elapsed = queryEndTime.getMillis() - queryStartTime.getMillis();
long scheduling = firstTaskStartTime - queryStartTime.getMillis() - planning;
long running = lastTaskEndTime - firstTaskStartTime;
long finishing = queryEndTime.getMillis() - lastTaskEndTime;
log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), max(elapsed, 0), max(planning, 0), max(scheduling, 0), max(running, 0), max(finishing, 0), queryStartTime, queryEndTime);
} catch (Exception e) {
log.error(e, "Error logging query timeline");
}
}
use of com.facebook.presto.operator.TaskStats in project presto by prestodb.
the class SqlTask method createTaskStatus.
private TaskStatus createTaskStatus(TaskHolder taskHolder) {
long taskStatusAgeInMilis = System.currentTimeMillis() - creationTimeInMillis;
// Always return a new TaskInfo with a larger version number;
// otherwise a client will not accept the update
long versionNumber = nextTaskInfoVersion.getAndIncrement();
TaskState state = taskStateMachine.getState();
List<ExecutionFailureInfo> failures = ImmutableList.of();
if (state == FAILED) {
failures = toFailures(taskStateMachine.getFailureCauses());
}
int queuedPartitionedDrivers = 0;
long queuedPartitionedSplitsWeight = 0L;
int runningPartitionedDrivers = 0;
long runningPartitionedSplitsWeight = 0L;
long physicalWrittenDataSizeInBytes = 0L;
long userMemoryReservationInBytes = 0L;
long systemMemoryReservationInBytes = 0L;
// TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
Set<Lifespan> completedDriverGroups = ImmutableSet.of();
long fullGcCount = 0;
long fullGcTimeInMillis = 0L;
long totalCpuTimeInNanos = 0L;
if (taskHolder.getFinalTaskInfo() != null) {
TaskStats taskStats = taskHolder.getFinalTaskInfo().getStats();
queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
queuedPartitionedSplitsWeight = taskStats.getQueuedPartitionedSplitsWeight();
runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
runningPartitionedSplitsWeight = taskStats.getRunningPartitionedSplitsWeight();
physicalWrittenDataSizeInBytes = taskStats.getPhysicalWrittenDataSizeInBytes();
userMemoryReservationInBytes = taskStats.getUserMemoryReservationInBytes();
systemMemoryReservationInBytes = taskStats.getSystemMemoryReservationInBytes();
fullGcCount = taskStats.getFullGcCount();
fullGcTimeInMillis = taskStats.getFullGcTimeInMillis();
totalCpuTimeInNanos = taskStats.getTotalCpuTimeInNanos();
} else if (taskHolder.getTaskExecution() != null) {
long physicalWrittenBytes = 0;
TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
queuedPartitionedSplitsWeight += pipelineStatus.getQueuedPartitionedSplitsWeight();
runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
runningPartitionedSplitsWeight += pipelineStatus.getRunningPartitionedSplitsWeight();
physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
totalCpuTimeInNanos += pipelineContext.getPipelineStats().getTotalCpuTimeInNanos();
}
physicalWrittenDataSizeInBytes = physicalWrittenBytes;
userMemoryReservationInBytes = taskContext.getMemoryReservation().toBytes();
systemMemoryReservationInBytes = taskContext.getSystemMemoryReservation().toBytes();
completedDriverGroups = taskContext.getCompletedDriverGroups();
fullGcCount = taskContext.getFullGcCount();
fullGcTimeInMillis = taskContext.getFullGcTime().toMillis();
}
return new TaskStatus(taskInstanceId.getUuidLeastSignificantBits(), taskInstanceId.getUuidMostSignificantBits(), versionNumber, state, location, completedDriverGroups, failures, queuedPartitionedDrivers, runningPartitionedDrivers, outputBuffer.getUtilization(), isOutputBufferOverutilized(), physicalWrittenDataSizeInBytes, userMemoryReservationInBytes, systemMemoryReservationInBytes, queryContext.getPeakNodeTotalMemory(), fullGcCount, fullGcTimeInMillis, totalCpuTimeInNanos, taskStatusAgeInMilis, queuedPartitionedSplitsWeight, runningPartitionedSplitsWeight);
}
use of com.facebook.presto.operator.TaskStats in project presto by prestodb.
the class SqlTask method createTaskInfo.
private TaskInfo createTaskInfo(TaskHolder taskHolder) {
TaskStats taskStats = getTaskStats(taskHolder);
Set<PlanNodeId> noMoreSplits = getNoMoreSplits(taskHolder);
MetadataUpdates metadataRequests = getMetadataUpdateRequests(taskHolder);
TaskStatus taskStatus = createTaskStatus(taskHolder);
return new TaskInfo(taskStateMachine.getTaskId(), taskStatus, lastHeartbeat.get(), outputBuffer.getInfo(), noMoreSplits, taskStats, needsPlan.get(), metadataRequests, nodeId);
}
use of com.facebook.presto.operator.TaskStats in project presto by prestodb.
the class SqlTask method getTaskStats.
private TaskStats getTaskStats(TaskHolder taskHolder) {
TaskInfo finalTaskInfo = taskHolder.getFinalTaskInfo();
if (finalTaskInfo != null) {
return finalTaskInfo.getStats();
}
SqlTaskExecution taskExecution = taskHolder.getTaskExecution();
if (taskExecution != null) {
return taskExecution.getTaskContext().getTaskStats();
}
// if the task completed without creation, set end time
DateTime endTime = taskStateMachine.getState().isDone() ? DateTime.now() : null;
return new TaskStats(taskStateMachine.getCreatedTime(), endTime);
}
Aggregations