Search in sources :

Example 1 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(List<StageInfo> stages, Metadata metadata, Session session) {
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = stages.stream().flatMap(stage -> getAllStages(Optional.of(stage)).stream()).collect(toImmutableList());
    for (StageInfo stageInfo : allStages) {
        Map<PlanNodeId, PlanNodeStats> aggregatedStats = new HashMap<>();
        List<PlanNodeStats> planNodeStats = stageInfo.getTasks().stream().map(TaskInfo::getStats).flatMap(taskStats -> getPlanNodeStats(taskStats).stream()).collect(toList());
        for (PlanNodeStats stats : planNodeStats) {
            aggregatedStats.merge(stats.getPlanNodeId(), stats, PlanNodeStats::merge);
        }
        builder.append(formatFragment(metadata, session, stageInfo.getPlan(), Optional.of(stageInfo.getStageStats()), Optional.of(aggregatedStats)));
    }
    return builder.toString();
}
Also used : ComparisonExpressionType(com.facebook.presto.sql.tree.ComparisonExpressionType) Arrays(java.util.Arrays) FrameBound(com.facebook.presto.sql.tree.FrameBound) DistinctLimitNode(com.facebook.presto.sql.planner.plan.DistinctLimitNode) OperatorStats(com.facebook.presto.operator.OperatorStats) StageInfo(com.facebook.presto.execution.StageInfo) MarkDistinctNode(com.facebook.presto.sql.planner.plan.MarkDistinctNode) Map(java.util.Map) IndexJoinNode(com.facebook.presto.sql.planner.plan.IndexJoinNode) ImmutableCollectors.toImmutableList(com.facebook.presto.util.ImmutableCollectors.toImmutableList) Double.max(java.lang.Double.max) StageInfo.getAllStages(com.facebook.presto.execution.StageInfo.getAllStages) StageStats(com.facebook.presto.execution.StageStats) RowNumberNode(com.facebook.presto.sql.planner.plan.RowNumberNode) Set(java.util.Set) VARCHAR(com.facebook.presto.spi.type.VarcharType.VARCHAR) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ExceptNode(com.facebook.presto.sql.planner.plan.ExceptNode) Domain(com.facebook.presto.spi.predicate.Domain) Stream(java.util.stream.Stream) OperatorNotFoundException(com.facebook.presto.metadata.OperatorNotFoundException) ApplyNode(com.facebook.presto.sql.planner.plan.ApplyNode) AssignUniqueId(com.facebook.presto.sql.planner.plan.AssignUniqueId) LimitNode(com.facebook.presto.sql.planner.plan.LimitNode) Lists.reverse(com.google.common.collect.Lists.reverse) Joiner(com.google.common.base.Joiner) Iterables(com.google.common.collect.Iterables) SINGLE_DISTRIBUTION(com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) Slice(io.airlift.slice.Slice) GroupIdNode(com.facebook.presto.sql.planner.plan.GroupIdNode) DomainUtils.simplifyDomain(com.facebook.presto.sql.planner.DomainUtils.simplifyDomain) DataSize.succinctBytes(io.airlift.units.DataSize.succinctBytes) ArrayList(java.util.ArrayList) TableScanNode(com.facebook.presto.sql.planner.plan.TableScanNode) Strings(com.google.common.base.Strings) Lists(com.google.common.collect.Lists) Type(com.facebook.presto.spi.type.Type) PlanNode(com.facebook.presto.sql.planner.plan.PlanNode) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) Functions(com.google.common.base.Functions) PipelineStats(com.facebook.presto.operator.PipelineStats) WindowNode(com.facebook.presto.sql.planner.plan.WindowNode) Session(com.facebook.presto.Session) Throwables(com.google.common.base.Throwables) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Range(com.facebook.presto.spi.predicate.Range) Math.sqrt(java.lang.Math.sqrt) ColumnHandle(com.facebook.presto.spi.ColumnHandle) FunctionInvoker(com.facebook.presto.sql.FunctionInvoker) SemiJoinNode(com.facebook.presto.sql.planner.plan.SemiJoinNode) FilterNode(com.facebook.presto.sql.planner.plan.FilterNode) TaskInfo(com.facebook.presto.execution.TaskInfo) Metadata(com.facebook.presto.metadata.Metadata) TaskStats(com.facebook.presto.operator.TaskStats) ExplainAnalyzeNode(com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode) RemoteSourceNode(com.facebook.presto.sql.planner.plan.RemoteSourceNode) SortNode(com.facebook.presto.sql.planner.plan.SortNode) MetadataDeleteNode(com.facebook.presto.sql.planner.plan.MetadataDeleteNode) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode) ValuesNode(com.facebook.presto.sql.planner.plan.ValuesNode) Duration(io.airlift.units.Duration) GraphvizPrinter(com.facebook.presto.util.GraphvizPrinter) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Collectors.toMap(java.util.stream.Collectors.toMap) Locale(java.util.Locale) TopNRowNumberNode(com.facebook.presto.sql.planner.plan.TopNRowNumberNode) IndexSourceNode(com.facebook.presto.sql.planner.plan.IndexSourceNode) TopNNode(com.facebook.presto.sql.planner.plan.TopNNode) Assignments(com.facebook.presto.sql.planner.plan.Assignments) NullableValue(com.facebook.presto.spi.predicate.NullableValue) DataSize.succinctDataSize(io.airlift.units.DataSize.succinctDataSize) FunctionCall(com.facebook.presto.sql.tree.FunctionCall) OutputNode(com.facebook.presto.sql.planner.plan.OutputNode) SymbolReference(com.facebook.presto.sql.tree.SymbolReference) ImmutableSet(com.google.common.collect.ImmutableSet) Scope(com.facebook.presto.sql.planner.plan.ExchangeNode.Scope) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) Iterables.getLast(com.google.common.collect.Iterables.getLast) Marker(com.facebook.presto.spi.predicate.Marker) Collectors(java.util.stream.Collectors) BinaryOperator(java.util.function.BinaryOperator) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) DataSize(io.airlift.units.DataSize) List(java.util.List) Window(com.facebook.presto.sql.tree.Window) SampleNode(com.facebook.presto.sql.planner.plan.SampleNode) Optional(java.util.Optional) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HashMap(java.util.HashMap) HashSet(java.util.HashSet) WindowFrame(com.facebook.presto.sql.tree.WindowFrame) PlanVisitor(com.facebook.presto.sql.planner.plan.PlanVisitor) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) Double.isFinite(java.lang.Double.isFinite) LinkedList(java.util.LinkedList) AggregationNode(com.facebook.presto.sql.planner.plan.AggregationNode) UnnestNode(com.facebook.presto.sql.planner.plan.UnnestNode) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) Collections.emptyMap(java.util.Collections.emptyMap) TableLayout(com.facebook.presto.metadata.TableLayout) CaseFormat(com.google.common.base.CaseFormat) Signature(com.facebook.presto.metadata.Signature) HashCollisionsInfo(com.facebook.presto.operator.HashCollisionsInfo) ComparisonExpression(com.facebook.presto.sql.tree.ComparisonExpression) DeleteNode(com.facebook.presto.sql.planner.plan.DeleteNode) ProjectNode(com.facebook.presto.sql.planner.plan.ProjectNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) UnionNode(com.facebook.presto.sql.planner.plan.UnionNode) Expression(com.facebook.presto.sql.tree.Expression) TableHandle(com.facebook.presto.metadata.TableHandle) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) BYTE(io.airlift.units.DataSize.Unit.BYTE) IntersectNode(com.facebook.presto.sql.planner.plan.IntersectNode) EnforceSingleRowNode(com.facebook.presto.sql.planner.plan.EnforceSingleRowNode) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) TaskInfo(com.facebook.presto.execution.TaskInfo) HashMap(java.util.HashMap) StageInfo(com.facebook.presto.execution.StageInfo)

Example 2 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime() == null ? 0 : queryStats.getTotalPlanningTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        // long lastSchedulingCompletion = 0;
        long firstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            // only consider leaf stages
            if (!stage.getSubStages().isEmpty()) {
                continue;
            }
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = queryEndTime.getMillis() - queryStartTime.getMillis();
        long scheduling = firstTaskStartTime - queryStartTime.getMillis() - planning;
        long running = lastTaskEndTime - firstTaskStartTime;
        long finishing = queryEndTime.getMillis() - lastTaskEndTime;
        log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), max(elapsed, 0), max(planning, 0), max(scheduling, 0), max(running, 0), max(finishing, 0), queryStartTime, queryEndTime);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}
Also used : TaskInfo(com.facebook.presto.execution.TaskInfo) QueryStats(com.facebook.presto.execution.QueryStats) StageInfo(com.facebook.presto.execution.StageInfo) TaskStats(com.facebook.presto.operator.TaskStats) DateTime(org.joda.time.DateTime) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) TransactionId(com.facebook.presto.transaction.TransactionId)

Example 3 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method createTaskStatus.

private TaskStatus createTaskStatus(TaskHolder taskHolder) {
    long taskStatusAgeInMilis = System.currentTimeMillis() - creationTimeInMillis;
    // Always return a new TaskInfo with a larger version number;
    // otherwise a client will not accept the update
    long versionNumber = nextTaskInfoVersion.getAndIncrement();
    TaskState state = taskStateMachine.getState();
    List<ExecutionFailureInfo> failures = ImmutableList.of();
    if (state == FAILED) {
        failures = toFailures(taskStateMachine.getFailureCauses());
    }
    int queuedPartitionedDrivers = 0;
    long queuedPartitionedSplitsWeight = 0L;
    int runningPartitionedDrivers = 0;
    long runningPartitionedSplitsWeight = 0L;
    long physicalWrittenDataSizeInBytes = 0L;
    long userMemoryReservationInBytes = 0L;
    long systemMemoryReservationInBytes = 0L;
    // TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
    Set<Lifespan> completedDriverGroups = ImmutableSet.of();
    long fullGcCount = 0;
    long fullGcTimeInMillis = 0L;
    long totalCpuTimeInNanos = 0L;
    if (taskHolder.getFinalTaskInfo() != null) {
        TaskStats taskStats = taskHolder.getFinalTaskInfo().getStats();
        queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
        queuedPartitionedSplitsWeight = taskStats.getQueuedPartitionedSplitsWeight();
        runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
        runningPartitionedSplitsWeight = taskStats.getRunningPartitionedSplitsWeight();
        physicalWrittenDataSizeInBytes = taskStats.getPhysicalWrittenDataSizeInBytes();
        userMemoryReservationInBytes = taskStats.getUserMemoryReservationInBytes();
        systemMemoryReservationInBytes = taskStats.getSystemMemoryReservationInBytes();
        fullGcCount = taskStats.getFullGcCount();
        fullGcTimeInMillis = taskStats.getFullGcTimeInMillis();
        totalCpuTimeInNanos = taskStats.getTotalCpuTimeInNanos();
    } else if (taskHolder.getTaskExecution() != null) {
        long physicalWrittenBytes = 0;
        TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
        for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
            PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
            queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
            queuedPartitionedSplitsWeight += pipelineStatus.getQueuedPartitionedSplitsWeight();
            runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
            runningPartitionedSplitsWeight += pipelineStatus.getRunningPartitionedSplitsWeight();
            physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
            totalCpuTimeInNanos += pipelineContext.getPipelineStats().getTotalCpuTimeInNanos();
        }
        physicalWrittenDataSizeInBytes = physicalWrittenBytes;
        userMemoryReservationInBytes = taskContext.getMemoryReservation().toBytes();
        systemMemoryReservationInBytes = taskContext.getSystemMemoryReservation().toBytes();
        completedDriverGroups = taskContext.getCompletedDriverGroups();
        fullGcCount = taskContext.getFullGcCount();
        fullGcTimeInMillis = taskContext.getFullGcTime().toMillis();
    }
    return new TaskStatus(taskInstanceId.getUuidLeastSignificantBits(), taskInstanceId.getUuidMostSignificantBits(), versionNumber, state, location, completedDriverGroups, failures, queuedPartitionedDrivers, runningPartitionedDrivers, outputBuffer.getUtilization(), isOutputBufferOverutilized(), physicalWrittenDataSizeInBytes, userMemoryReservationInBytes, systemMemoryReservationInBytes, queryContext.getPeakNodeTotalMemory(), fullGcCount, fullGcTimeInMillis, totalCpuTimeInNanos, taskStatusAgeInMilis, queuedPartitionedSplitsWeight, runningPartitionedSplitsWeight);
}
Also used : PipelineStatus(com.facebook.presto.operator.PipelineStatus) TaskContext(com.facebook.presto.operator.TaskContext) PipelineContext(com.facebook.presto.operator.PipelineContext) TaskStats(com.facebook.presto.operator.TaskStats)

Example 4 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method createTaskInfo.

private TaskInfo createTaskInfo(TaskHolder taskHolder) {
    TaskStats taskStats = getTaskStats(taskHolder);
    Set<PlanNodeId> noMoreSplits = getNoMoreSplits(taskHolder);
    MetadataUpdates metadataRequests = getMetadataUpdateRequests(taskHolder);
    TaskStatus taskStatus = createTaskStatus(taskHolder);
    return new TaskInfo(taskStateMachine.getTaskId(), taskStatus, lastHeartbeat.get(), outputBuffer.getInfo(), noMoreSplits, taskStats, needsPlan.get(), metadataRequests, nodeId);
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) MetadataUpdates(com.facebook.presto.metadata.MetadataUpdates) TaskStats(com.facebook.presto.operator.TaskStats)

Example 5 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class SqlTask method getTaskStats.

private TaskStats getTaskStats(TaskHolder taskHolder) {
    TaskInfo finalTaskInfo = taskHolder.getFinalTaskInfo();
    if (finalTaskInfo != null) {
        return finalTaskInfo.getStats();
    }
    SqlTaskExecution taskExecution = taskHolder.getTaskExecution();
    if (taskExecution != null) {
        return taskExecution.getTaskContext().getTaskStats();
    }
    // if the task completed without creation, set end time
    DateTime endTime = taskStateMachine.getState().isDone() ? DateTime.now() : null;
    return new TaskStats(taskStateMachine.getCreatedTime(), endTime);
}
Also used : TaskStats(com.facebook.presto.operator.TaskStats) DateTime(org.joda.time.DateTime)

Aggregations

TaskStats (com.facebook.presto.operator.TaskStats)12 TaskInfo (com.facebook.presto.execution.TaskInfo)5 HashSet (java.util.HashSet)4 StageInfo (com.facebook.presto.execution.StageInfo)3 BlockedReason (com.facebook.presto.operator.BlockedReason)3 OperatorStats (com.facebook.presto.operator.OperatorStats)3 PipelineStats (com.facebook.presto.operator.PipelineStats)3 Session (com.facebook.presto.Session)2 QueryStats (com.facebook.presto.execution.QueryStats)2 TaskStatus (com.facebook.presto.execution.TaskStatus)2 TaskContext (com.facebook.presto.operator.TaskContext)2 QueryId (com.facebook.presto.spi.QueryId)2 HashMap (java.util.HashMap)2 DateTime (org.joda.time.DateTime)2 Logger (com.facebook.airlift.log.Logger)1 CpuTimer (com.facebook.airlift.stats.CpuTimer)1 CpuDuration (com.facebook.airlift.stats.CpuTimer.CpuDuration)1 GarbageCollectionNotificationInfo (com.facebook.airlift.stats.GarbageCollectionNotificationInfo)1 TestingGcMonitor (com.facebook.airlift.stats.TestingGcMonitor)1 RuntimeStats (com.facebook.presto.common.RuntimeStats)1