Examples with BlockedReason io.prestosql.operator.BlockedReason used on opensource projects

Example 1 with BlockedReason

use of io.prestosql.operator.BlockedReason in project hetu-core by openlookeng.

the class TestStateUpdater method createBasicQueryInfo.

private BasicQueryInfo createBasicQueryInfo() {
    QueryInfo queryInfo = Mockito.mock(QueryInfo.class);
    when(queryInfo.getQueryStats()).then(new Returns(Mockito.mock(QueryStats.class)));
    Duration mockInterval = new Duration(MINIMUM_UPDATE_INTERVAL, MILLISECONDS);
    when(queryInfo.getQueryStats().getQueuedTime()).then(new Returns(mockInterval));
    when(queryInfo.getQueryStats().getElapsedTime()).then(new Returns(mockInterval));
    when(queryInfo.getQueryStats().getExecutionTime()).then(new Returns(mockInterval));
    when(queryInfo.getQueryStats().getRawInputDataSize()).then(new Returns(Mockito.mock(DataSize.class)));
    String mockQueryId = MOCK_QUERY_ID;
    QueryId queryId = new QueryId(mockQueryId);
    when(queryInfo.getQueryId()).then(new Returns(queryId));
    SessionRepresentation sessionRepresentation = TEST_SESSION.toSessionRepresentation();
    when(queryInfo.getSession()).then(new Returns(sessionRepresentation));
    ResourceGroupId resourceGroupId = new ResourceGroupId(GLOBAL_RESOURCE_ID);
    Optional<ResourceGroupId> optionalResourceGroupId = Optional.of(resourceGroupId);
    when(queryInfo.getResourceGroupId()).then(new Returns(optionalResourceGroupId));
    when(queryInfo.getState()).then(new Returns(QueryState.FINISHED));
    URI mockURI = URI.create(URI_LOCALHOST);
    when(queryInfo.getSelf()).then(new Returns(mockURI));
    String mockQuery = QUERY_STRING;
    when(queryInfo.getQuery()).then(new Returns(mockQuery));
    Optional<String> preparedQuery = Optional.of(PREPARED_QUERY_MOCK_DATA);
    when(queryInfo.getPreparedQuery()).then(new Returns(preparedQuery));
    ErrorCode errorCode = new ErrorCode(ERROR_CODE_VALUE_INDEX_TIME_NO_INVOCATION, ERROR_CODE_TEST, USER_ERROR);
    when(queryInfo.getErrorCode()).then(new Returns(errorCode));
    Set<BlockedReason> setBlockedReason = new HashSet<>();
    setBlockedReason.add(BlockedReason.WAITING_FOR_MEMORY);
    when(queryInfo.getQueryStats().getBlockedReasons()).then(new Returns(setBlockedReason));
    when(queryInfo.getQueryStats().getProgressPercentage()).then(new Returns(OptionalDouble.empty()));
    BasicQueryInfo basicQueryInfo = new BasicQueryInfo(queryInfo);
    return basicQueryInfo;
}

Example 2 with BlockedReason

use of io.prestosql.operator.BlockedReason in project hetu-core by openlookeng.

the class StageStateMachine method getBasicStageStats.

public BasicStageStats getBasicStageStats(Supplier<Iterable<TaskInfo>> taskInfosSupplier) {
    Optional<StageInfo> localFinalStageInfo = this.finalStageInfo.get();
    if (localFinalStageInfo.isPresent()) {
        return localFinalStageInfo.get().getStageStats().toBasicStageStats(localFinalStageInfo.get().getState());
    }
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    // Snapshot: RESCHEDULING, although a done state for stage, should not be deemed as "scheduled".
    boolean isScheduled = (state == RUNNING) || state.isDone() && state != RESCHEDULING;
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long physicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long internalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        long taskUserMemory = taskStats.getUserMemoryReservation().toBytes();
        long taskSystemMemory = taskStats.getSystemMemoryReservation().toBytes();
        long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes();
        userMemoryReservation += taskUserMemory;
        totalMemoryReservation += taskUserMemory + taskSystemMemory + taskRevocableMemory;
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        physicalInputDataSize += taskStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += taskStats.getPhysicalInputPositions();
        internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
        if (fragment.getPartitionedSourceNodes().stream().anyMatch(TableScanNode.class::isInstance)) {
            rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
            rawInputPositions += taskStats.getRawInputPositions();
        }
    }
    OptionalDouble progressPercentage = OptionalDouble.empty();
    if (isScheduled && totalDrivers != 0) {
        progressPercentage = OptionalDouble.of(min(100, (completedDrivers * 100.0) / totalDrivers));
    }
    return new BasicStageStats(isScheduled, totalDrivers, queuedDrivers, runningDrivers, completedDrivers, succinctBytes(physicalInputDataSize), physicalInputPositions, succinctBytes(internalNetworkInputDataSize), internalNetworkInputPositions, succinctBytes(rawInputDataSize), rawInputPositions, cumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), fullyBlocked, blockedReasons, progressPercentage);
}

Example 3 with BlockedReason

use of io.prestosql.operator.BlockedReason in project hetu-core by openlookeng.

the class BasicStageStats method aggregateBasicStageStats.

public static BasicStageStats aggregateBasicStageStats(Iterable<BasicStageStats> stages) {
    int localTotalDrivers = 0;
    int localQueuedDrivers = 0;
    int localRunningDrivers = 0;
    int localCompletedDrivers = 0;
    long localCumulativeUserMemory = 0;
    long localUserMemoryReservation = 0;
    long localTotalMemoryReservation = 0;
    long totalScheduledTimeMillis = 0;
    long localTotalCpuTime = 0;
    long localPhysicalInputDataSize = 0;
    long localPhysicalInputPositions = 0;
    long localInternalNetworkInputDataSize = 0;
    long localInternalNetworkInputPositions = 0;
    long localRawInputDataSize = 0;
    long localRawInputPositions = 0;
    boolean localScheduled = true;
    boolean localFullyBlocked = true;
    Set<BlockedReason> localBlockedReasons = new HashSet<>();
    for (BasicStageStats stageStats : stages) {
        localTotalDrivers += stageStats.getTotalDrivers();
        localQueuedDrivers += stageStats.getQueuedDrivers();
        localRunningDrivers += stageStats.getRunningDrivers();
        localCompletedDrivers += stageStats.getCompletedDrivers();
        localCumulativeUserMemory += stageStats.getCumulativeUserMemory();
        localUserMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
        localTotalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
        totalScheduledTimeMillis += stageStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        localTotalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS);
        localScheduled &= stageStats.isScheduled();
        localFullyBlocked &= stageStats.isFullyBlocked();
        localBlockedReasons.addAll(stageStats.getBlockedReasons());
        localPhysicalInputDataSize += stageStats.getPhysicalInputDataSize().toBytes();
        localPhysicalInputPositions += stageStats.getPhysicalInputPositions();
        localInternalNetworkInputDataSize += stageStats.getInternalNetworkInputDataSize().toBytes();
        localInternalNetworkInputPositions += stageStats.getInternalNetworkInputPositions();
        localRawInputDataSize += stageStats.getRawInputDataSize().toBytes();
        localRawInputPositions += stageStats.getRawInputPositions();
    }
    OptionalDouble localProgressPercentage = OptionalDouble.empty();
    if (localScheduled && localTotalDrivers != 0) {
        localProgressPercentage = OptionalDouble.of(min(100, (localCompletedDrivers * 100.0) / localTotalDrivers));
    }
    return new BasicStageStats(localScheduled, localTotalDrivers, localQueuedDrivers, localRunningDrivers, localCompletedDrivers, succinctBytes(localPhysicalInputDataSize), localPhysicalInputPositions, succinctBytes(localInternalNetworkInputDataSize), localInternalNetworkInputPositions, succinctBytes(localRawInputDataSize), localRawInputPositions, localCumulativeUserMemory, succinctBytes(localUserMemoryReservation), succinctBytes(localTotalMemoryReservation), new Duration(localTotalCpuTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalScheduledTimeMillis, MILLISECONDS).convertToMostSuccinctTimeUnit(), localFullyBlocked, localBlockedReasons, localProgressPercentage);
}

Example 4 with BlockedReason

use of io.prestosql.operator.BlockedReason in project hetu-core by openlookeng.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier) {
    Optional<StageInfo> localFinalStageInfo = this.finalStageInfo.get();
    if (localFinalStageInfo.isPresent()) {
        return localFinalStageInfo.get();
    }
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long peakUserMemoryReservation = peakUserMemory.get();
    long peakRevocableMemoryReservation = peakRevocableMemory.get();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long internalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long bufferedDataSize = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    int fullGcCount = 0;
    int fullGcTaskCount = 0;
    int minFullGcSec = 0;
    int maxFullGcSec = 0;
    int totalFullGcSec = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        blockedDrivers += taskStats.getBlockedDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        long taskUserMemory = taskStats.getUserMemoryReservation().toBytes();
        long taskSystemMemory = taskStats.getSystemMemoryReservation().toBytes();
        long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes();
        userMemoryReservation += taskUserMemory;
        revocableMemoryReservation += taskRevocableMemory;
        totalMemoryReservation += taskUserMemory + taskSystemMemory + taskRevocableMemory;
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        physicalInputDataSize += taskStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += taskStats.getPhysicalInputPositions();
        internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSize().toBytes();
        fullGcCount += taskStats.getFullGcCount();
        fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
        int gcSec = toIntExact(taskStats.getFullGcTime().roundTo(SECONDS));
        totalFullGcSec += gcSec;
        minFullGcSec = min(minFullGcSec, gcSec);
        maxFullGcSec = max(maxFullGcSec, gcSec);
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(peakUserMemoryReservation), succinctBytes(peakRevocableMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(physicalInputDataSize), physicalInputPositions, succinctBytes(internalNetworkInputDataSize), internalNetworkInputPositions, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), new StageGcStatistics(stageId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, ImmutableList.of(), tables, failureInfo);
}

Example 5 with BlockedReason

use of io.prestosql.operator.BlockedReason in project hetu-core by openlookeng.

the class QueryStateMachine method getQueryStats.

private QueryStats getQueryStats(Optional<StageInfo> rootStage) {
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long internalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageStats stageStats = stageInfo.getStageStats();
        totalTasks += stageStats.getTotalTasks();
        runningTasks += stageStats.getRunningTasks();
        completedTasks += stageStats.getCompletedTasks();
        totalDrivers += stageStats.getTotalDrivers();
        queuedDrivers += stageStats.getQueuedDrivers();
        runningDrivers += stageStats.getRunningDrivers();
        blockedDrivers += stageStats.getBlockedDrivers();
        completedDrivers += stageStats.getCompletedDrivers();
        cumulativeUserMemory += stageStats.getCumulativeUserMemory();
        userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
        revocableMemoryReservation += stageStats.getRevocableMemoryReservation().toBytes();
        totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
        totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        totalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS);
        totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(MILLISECONDS);
        if (!stageInfo.getState().isDone()) {
            fullyBlocked &= stageStats.isFullyBlocked();
            blockedReasons.addAll(stageStats.getBlockedReasons());
        }
        physicalInputDataSize += stageStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += stageStats.getPhysicalInputPositions();
        internalNetworkInputDataSize += stageStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += stageStats.getInternalNetworkInputPositions();
        PlanFragment plan = stageInfo.getPlan();
        if (plan != null && plan.getPartitionedSourceNodes().stream().anyMatch(TableScanNode.class::isInstance)) {
            rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
            rawInputPositions += stageStats.getRawInputPositions();
            processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
            processedInputPositions += stageStats.getProcessedInputPositions();
        }
        physicalWrittenDataSize += stageStats.getPhysicalWrittenDataSize().toBytes();
        stageGcStatistics.add(stageStats.getGcInfo());
        completeInfo = completeInfo && stageInfo.isCompleteInfo();
        operatorStatsSummary.addAll(stageInfo.getStageStats().getOperatorSummaries());
    }
    if (rootStage.isPresent()) {
        StageStats outputStageStats = rootStage.get().getStageStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
    }
    boolean isScheduled = isScheduled(rootStage);
    return new QueryStats(queryStateTimer.getCreateTime(), getExecutionStartTime().orElse(null), getLastHeartbeat(), getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getDistributedPlanningTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getLogicalPlanningTime(), queryStateTimer.getSyntaxAnalysisTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(getPeakUserMemoryInBytes()), succinctBytes(getPeakRevocableMemoryInBytes()), succinctBytes(getPeakTotalMemoryInBytes()), succinctBytes(getPeakTaskUserMemory()), succinctBytes(getPeakTaskRevocableMemory()), succinctBytes(getPeakTaskTotalMemory()), isScheduled, new Duration(totalScheduledTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), fullyBlocked, blockedReasons, succinctBytes(physicalInputDataSize), physicalInputPositions, succinctBytes(internalNetworkInputDataSize), internalNetworkInputPositions, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), stageGcStatistics.build(), operatorStatsSummary.build());
}