Examples with BlockedReason com.facebook.presto.operator.BlockedReason used on opensource projects

Example 1 with BlockedReason

use of com.facebook.presto.operator.BlockedReason in project presto by prestodb.

the class QueryStateMachine method getQueryInfo.

public QueryInfo getQueryInfo(Optional<StageInfo> rootStage) {
    // Query state must be captured first in order to provide a
    // correct view of the query.  For example, building this
    // information, the query could finish, and the task states would
    // never be visible.
    QueryState state = queryState.get();
    Duration elapsedTime;
    if (endNanos.get() != 0) {
        elapsedTime = new Duration(endNanos.get() - createNanos, NANOSECONDS);
    } else {
        elapsedTime = nanosSince(createNanos);
    }
    // don't report failure info is query is marked as success
    FailureInfo failureInfo = null;
    ErrorCode errorCode = null;
    if (state == FAILED) {
        ExecutionFailureInfo failureCause = this.failureCause.get();
        if (failureCause != null) {
            failureInfo = failureCause.toFailureInfo();
            errorCode = failureCause.getErrorCode();
        }
    }
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeMemory = 0;
    long totalMemoryReservation = 0;
    long peakMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalUserTime = 0;
    long totalBlockedTime = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageStats stageStats = stageInfo.getStageStats();
        totalTasks += stageStats.getTotalTasks();
        runningTasks += stageStats.getRunningTasks();
        completedTasks += stageStats.getCompletedTasks();
        totalDrivers += stageStats.getTotalDrivers();
        queuedDrivers += stageStats.getQueuedDrivers();
        runningDrivers += stageStats.getRunningDrivers();
        completedDrivers += stageStats.getCompletedDrivers();
        cumulativeMemory += stageStats.getCumulativeMemory();
        totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
        peakMemoryReservation = getPeakMemoryInBytes();
        totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += stageStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalUserTime += stageStats.getTotalUserTime().roundTo(NANOSECONDS);
        totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!stageInfo.getState().isDone()) {
            fullyBlocked &= stageStats.isFullyBlocked();
            blockedReasons.addAll(stageStats.getBlockedReasons());
        }
        PlanFragment plan = stageInfo.getPlan();
        if (plan != null && plan.getPartitionedSourceNodes().stream().anyMatch(TableScanNode.class::isInstance)) {
            rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
            rawInputPositions += stageStats.getRawInputPositions();
            processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
            processedInputPositions += stageStats.getProcessedInputPositions();
        }
        completeInfo = completeInfo && stageInfo.isCompleteInfo();
        operatorStatsSummary.addAll(stageInfo.getStageStats().getOperatorSummaries());
    }
    if (rootStage.isPresent()) {
        StageStats outputStageStats = rootStage.get().getStageStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
    }
    QueryStats queryStats = new QueryStats(createTime, executionStartTime.get(), lastHeartbeat.get(), endTime.get(), elapsedTime.convertToMostSuccinctTimeUnit(), queuedTime.get(), analysisTime.get(), distributedPlanningTime.get(), totalPlanningTime.get(), finishingTime.get(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, cumulativeMemory, succinctBytes(totalMemoryReservation), succinctBytes(peakMemoryReservation), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalUserTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), fullyBlocked, blockedReasons, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, operatorStatsSummary.build());
    return new QueryInfo(queryId, session.toSessionRepresentation(), state, memoryPool.get().getId(), isScheduled(rootStage), self, outputFieldNames.get(), query, queryStats, setSessionProperties, resetSessionProperties, addedPreparedStatements, deallocatedPreparedStatements, Optional.ofNullable(startedTransactionId.get()), clearTransactionId.get(), updateType.get(), rootStage, failureInfo, errorCode, inputs.get(), output.get(), completeInfo, getResourceGroup().map(ResourceGroupId::toString));
}

Example 2 with BlockedReason

use of com.facebook.presto.operator.BlockedReason in project presto by prestodb.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier, Supplier<Iterable<StageInfo>> subStageInfosSupplier) {
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    List<StageInfo> subStageInfos = ImmutableList.copyOf(subStageInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeMemory = 0;
    long totalMemoryReservation = 0;
    long peakMemoryReservation = getPeakMemoryInBytes();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalUserTime = 0;
    long totalBlockedTime = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeMemory += taskStats.getCumulativeMemory();
        totalMemoryReservation += taskStats.getMemoryReservation().toBytes();
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalUserTime += taskStats.getTotalUserTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), scheduleTaskDistribution.snapshot(), addSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, cumulativeMemory, succinctBytes(totalMemoryReservation), succinctBytes(peakMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalUserTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, subStageInfos, failureInfo);
}