Examples with TaskStats com.facebook.presto.operator.TaskStats used on opensource projects

Example 6 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class TaskSystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(TASK_TABLE);
    for (TaskInfo taskInfo : taskManager.getAllTaskInfo()) {
        TaskStats stats = taskInfo.getStats();
        TaskStatus taskStatus = taskInfo.getTaskStatus();
        table.addRow(nodeId, taskStatus.getTaskId().toString(), taskStatus.getTaskId().getStageId().toString(), taskStatus.getTaskId().getQueryId().toString(), taskStatus.getState().toString(), (long) stats.getTotalDrivers(), (long) stats.getQueuedDrivers(), (long) stats.getRunningDrivers(), (long) stats.getCompletedDrivers(), toMillis(stats.getTotalScheduledTime()), toMillis(stats.getTotalCpuTime()), toMillis(stats.getTotalUserTime()), toMillis(stats.getTotalBlockedTime()), toBytes(stats.getRawInputDataSize()), stats.getRawInputPositions(), toBytes(stats.getProcessedInputDataSize()), stats.getProcessedInputPositions(), toBytes(stats.getOutputDataSize()), stats.getOutputPositions(), toTimeStamp(stats.getCreateTime()), toTimeStamp(stats.getFirstStartTime()), toTimeStamp(taskInfo.getLastHeartbeat()), toTimeStamp(stats.getEndTime()));
    }
    return table.build().cursor();
}

Example 7 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier, Supplier<Iterable<StageInfo>> subStageInfosSupplier) {
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    List<StageInfo> subStageInfos = ImmutableList.copyOf(subStageInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeMemory = 0;
    long totalMemoryReservation = 0;
    long peakMemoryReservation = getPeakMemoryInBytes();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalUserTime = 0;
    long totalBlockedTime = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeMemory += taskStats.getCumulativeMemory();
        totalMemoryReservation += taskStats.getMemoryReservation().toBytes();
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalUserTime += taskStats.getTotalUserTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), scheduleTaskDistribution.snapshot(), addSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, cumulativeMemory, succinctBytes(totalMemoryReservation), succinctBytes(peakMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalUserTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, subStageInfos, failureInfo);
}

Example 8 with TaskStats

use of com.facebook.presto.operator.TaskStats in project presto by prestodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    ExecutorService executor = localQueryRunner.getExecutor();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), memoryPool, systemMemoryPool, executor).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), executor), session, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSize().toBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSize().toBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}