Examples with OperatorStats io.prestosql.operator.OperatorStats used on opensource projects

Example 6 with OperatorStats

use of io.prestosql.operator.OperatorStats in project boostkit-bigdata by kunpengcompute.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method searchScanFilterAndProjectOperatorStats.

private OperatorStats searchScanFilterAndProjectOperatorStats(QueryId queryId, String tableName) {
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    Plan plan = runner.getQueryPlan(queryId);
    PlanNodeId nodeId = PlanNodeSearcher.searchFrom(plan.getRoot()).where(node -> {
        if (!(node instanceof ProjectNode)) {
            return false;
        }
        ProjectNode projectNode = (ProjectNode) node;
        FilterNode filterNode = (FilterNode) projectNode.getSource();
        TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
        return tableName.equals(tableScanNode.getTable().getConnectorHandle().toString());
    }).findOnlyElement().getId();
    return runner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getQueryStats().getOperatorSummaries().stream().filter(summary -> nodeId.equals(summary.getPlanNodeId())).collect(MoreCollectors.onlyElement());
}

Example 7 with OperatorStats

use of io.prestosql.operator.OperatorStats in project hetu-core by openlookeng.

the class StageStateMachine method getStageInfo.

public StageInfo getStageInfo(Supplier<Iterable<TaskInfo>> taskInfosSupplier) {
    Optional<StageInfo> localFinalStageInfo = this.finalStageInfo.get();
    if (localFinalStageInfo.isPresent()) {
        return localFinalStageInfo.get();
    }
    // stage state must be captured first in order to provide a
    // consistent view of the stage. For example, building this
    // information, the stage could finish, and the task states would
    // never be visible.
    StageState state = stageState.get();
    List<TaskInfo> taskInfos = ImmutableList.copyOf(taskInfosSupplier.get());
    int totalTasks = taskInfos.size();
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long peakUserMemoryReservation = peakUserMemory.get();
    long peakRevocableMemoryReservation = peakRevocableMemory.get();
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long internalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long bufferedDataSize = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    int fullGcCount = 0;
    int fullGcTaskCount = 0;
    int minFullGcSec = 0;
    int maxFullGcSec = 0;
    int totalFullGcSec = 0;
    boolean fullyBlocked = true;
    Set<BlockedReason> blockedReasons = new HashSet<>();
    Map<String, OperatorStats> operatorToStats = new HashMap<>();
    for (TaskInfo taskInfo : taskInfos) {
        TaskState taskState = taskInfo.getTaskStatus().getState();
        if (taskState.isDone()) {
            completedTasks++;
        } else {
            runningTasks++;
        }
        TaskStats taskStats = taskInfo.getStats();
        totalDrivers += taskStats.getTotalDrivers();
        queuedDrivers += taskStats.getQueuedDrivers();
        runningDrivers += taskStats.getRunningDrivers();
        blockedDrivers += taskStats.getBlockedDrivers();
        completedDrivers += taskStats.getCompletedDrivers();
        cumulativeUserMemory += taskStats.getCumulativeUserMemory();
        long taskUserMemory = taskStats.getUserMemoryReservation().toBytes();
        long taskSystemMemory = taskStats.getSystemMemoryReservation().toBytes();
        long taskRevocableMemory = taskStats.getRevocableMemoryReservation().toBytes();
        userMemoryReservation += taskUserMemory;
        revocableMemoryReservation += taskRevocableMemory;
        totalMemoryReservation += taskUserMemory + taskSystemMemory + taskRevocableMemory;
        totalScheduledTime += taskStats.getTotalScheduledTime().roundTo(NANOSECONDS);
        totalCpuTime += taskStats.getTotalCpuTime().roundTo(NANOSECONDS);
        totalBlockedTime += taskStats.getTotalBlockedTime().roundTo(NANOSECONDS);
        if (!taskState.isDone()) {
            fullyBlocked &= taskStats.isFullyBlocked();
            blockedReasons.addAll(taskStats.getBlockedReasons());
        }
        physicalInputDataSize += taskStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += taskStats.getPhysicalInputPositions();
        internalNetworkInputDataSize += taskStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += taskStats.getInternalNetworkInputPositions();
        rawInputDataSize += taskStats.getRawInputDataSize().toBytes();
        rawInputPositions += taskStats.getRawInputPositions();
        processedInputDataSize += taskStats.getProcessedInputDataSize().toBytes();
        processedInputPositions += taskStats.getProcessedInputPositions();
        bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes();
        outputDataSize += taskStats.getOutputDataSize().toBytes();
        outputPositions += taskStats.getOutputPositions();
        physicalWrittenDataSize += taskStats.getPhysicalWrittenDataSize().toBytes();
        fullGcCount += taskStats.getFullGcCount();
        fullGcTaskCount += taskStats.getFullGcCount() > 0 ? 1 : 0;
        int gcSec = toIntExact(taskStats.getFullGcTime().roundTo(SECONDS));
        totalFullGcSec += gcSec;
        minFullGcSec = min(minFullGcSec, gcSec);
        maxFullGcSec = max(maxFullGcSec, gcSec);
        for (PipelineStats pipeline : taskStats.getPipelines()) {
            for (OperatorStats operatorStats : pipeline.getOperatorSummaries()) {
                String id = pipeline.getPipelineId() + "." + operatorStats.getOperatorId();
                operatorToStats.compute(id, (k, v) -> v == null ? operatorStats : v.add(operatorStats));
            }
        }
    }
    StageStats stageStats = new StageStats(schedulingComplete.get(), getSplitDistribution.snapshot(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(peakUserMemoryReservation), succinctBytes(peakRevocableMemoryReservation), succinctDuration(totalScheduledTime, NANOSECONDS), succinctDuration(totalCpuTime, NANOSECONDS), succinctDuration(totalBlockedTime, NANOSECONDS), fullyBlocked && runningTasks > 0, blockedReasons, succinctBytes(physicalInputDataSize), physicalInputPositions, succinctBytes(internalNetworkInputDataSize), internalNetworkInputPositions, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(bufferedDataSize), succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), new StageGcStatistics(stageId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, maxFullGcSec, totalFullGcSec, (int) (1.0 * totalFullGcSec / fullGcCount)), ImmutableList.copyOf(operatorToStats.values()));
    ExecutionFailureInfo failureInfo = null;
    if (state == FAILED) {
        failureInfo = failureCause.get();
    }
    return new StageInfo(stageId, state, location, fragment, fragment.getTypes(), stageStats, taskInfos, ImmutableList.of(), tables, failureInfo);
}

Example 8 with OperatorStats

use of io.prestosql.operator.OperatorStats in project hetu-core by openlookeng.

the class QueryStateMachine method getQueryStats.

private QueryStats getQueryStats(Optional<StageInfo> rootStage) {
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    long cumulativeUserMemory = 0;
    long userMemoryReservation = 0;
    long revocableMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long totalBlockedTime = 0;
    long physicalInputDataSize = 0;
    long physicalInputPositions = 0;
    long internalNetworkInputDataSize = 0;
    long internalNetworkInputPositions = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long physicalWrittenDataSize = 0;
    ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageStats stageStats = stageInfo.getStageStats();
        totalTasks += stageStats.getTotalTasks();
        runningTasks += stageStats.getRunningTasks();
        completedTasks += stageStats.getCompletedTasks();
        totalDrivers += stageStats.getTotalDrivers();
        queuedDrivers += stageStats.getQueuedDrivers();
        runningDrivers += stageStats.getRunningDrivers();
        blockedDrivers += stageStats.getBlockedDrivers();
        completedDrivers += stageStats.getCompletedDrivers();
        cumulativeUserMemory += stageStats.getCumulativeUserMemory();
        userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
        revocableMemoryReservation += stageStats.getRevocableMemoryReservation().toBytes();
        totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();
        totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        totalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS);
        totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(MILLISECONDS);
        if (!stageInfo.getState().isDone()) {
            fullyBlocked &= stageStats.isFullyBlocked();
            blockedReasons.addAll(stageStats.getBlockedReasons());
        }
        physicalInputDataSize += stageStats.getPhysicalInputDataSize().toBytes();
        physicalInputPositions += stageStats.getPhysicalInputPositions();
        internalNetworkInputDataSize += stageStats.getInternalNetworkInputDataSize().toBytes();
        internalNetworkInputPositions += stageStats.getInternalNetworkInputPositions();
        PlanFragment plan = stageInfo.getPlan();
        if (plan != null && plan.getPartitionedSourceNodes().stream().anyMatch(TableScanNode.class::isInstance)) {
            rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
            rawInputPositions += stageStats.getRawInputPositions();
            processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
            processedInputPositions += stageStats.getProcessedInputPositions();
        }
        physicalWrittenDataSize += stageStats.getPhysicalWrittenDataSize().toBytes();
        stageGcStatistics.add(stageStats.getGcInfo());
        completeInfo = completeInfo && stageInfo.isCompleteInfo();
        operatorStatsSummary.addAll(stageInfo.getStageStats().getOperatorSummaries());
    }
    if (rootStage.isPresent()) {
        StageStats outputStageStats = rootStage.get().getStageStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
    }
    boolean isScheduled = isScheduled(rootStage);
    return new QueryStats(queryStateTimer.getCreateTime(), getExecutionStartTime().orElse(null), getLastHeartbeat(), getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getDistributedPlanningTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getLogicalPlanningTime(), queryStateTimer.getSyntaxAnalysisTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, succinctBytes(userMemoryReservation), succinctBytes(revocableMemoryReservation), succinctBytes(totalMemoryReservation), succinctBytes(getPeakUserMemoryInBytes()), succinctBytes(getPeakRevocableMemoryInBytes()), succinctBytes(getPeakTotalMemoryInBytes()), succinctBytes(getPeakTaskUserMemory()), succinctBytes(getPeakTaskRevocableMemory()), succinctBytes(getPeakTaskTotalMemory()), isScheduled, new Duration(totalScheduledTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalBlockedTime, MILLISECONDS).convertToMostSuccinctTimeUnit(), fullyBlocked, blockedReasons, succinctBytes(physicalInputDataSize), physicalInputPositions, succinctBytes(internalNetworkInputDataSize), internalNetworkInputPositions, succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, succinctBytes(physicalWrittenDataSize), stageGcStatistics.build(), operatorStatsSummary.build());
}

Example 9 with OperatorStats

use of io.prestosql.operator.OperatorStats in project boostkit-bigdata by kunpengcompute.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testJoinWithSelectiveBuildSide.

@Test
public void testJoinWithSelectiveBuildSide() {
    Session session = Session.builder(getSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, FeaturesConfig.JoinDistributionType.BROADCAST.name()).build();
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    ResultWithQueryId<MaterializedResult> result = runner.executeWithQueryId(session, "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey AND orders.custkey = 1");
    assertGreaterThan(result.getResult().getRowCount(), 0);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), "tpch:lineitem");
    // Probe side may be partially scanned, depending on the drivers' scheduling:
    assertLessThanOrEqual(probeStats.getInputPositions(), countRows("lineitem"));
}

Example 10 with OperatorStats

use of io.prestosql.operator.OperatorStats in project boostkit-bigdata by kunpengcompute.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testJoinWithEmptyBuildSide.

@Test
public void testJoinWithEmptyBuildSide() {
    Session session = Session.builder(getSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, FeaturesConfig.JoinDistributionType.BROADCAST.name()).build();
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    ResultWithQueryId<MaterializedResult> result = runner.executeWithQueryId(session, "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey AND orders.totalprice = 123.4567");
    assertEquals(result.getResult().getRowCount(), 0);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), "tpch:lineitem");
    // Probe-side is not scanned at all, due to dynamic filtering:
    assertEquals(probeStats.getInputPositions(), 0L);
}