Examples with PlanFragment com.facebook.presto.sql.planner.PlanFragment used on opensource projects

Example 21 with PlanFragment

use of com.facebook.presto.sql.planner.PlanFragment in project presto by prestodb.

the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.

@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
    // determine output fragment
    Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
    Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
    Visitor visitor = new Visitor(fragments);
    rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId()));
    return visitor.getSchedulerOrder();
}

Example 22 with PlanFragment

use of com.facebook.presto.sql.planner.PlanFragment in project presto by prestodb.

the class QueryStats method create.

public static QueryStats create(QueryStateTimer queryStateTimer, Optional<StageInfo> rootStage, int peakRunningTasks, DataSize peakUserMemoryReservation, DataSize peakTotalMemoryReservation, DataSize peakTaskUserMemory, DataSize peakTaskTotalMemory, DataSize peakNodeTotalMemory, RuntimeStats runtimeStats) {
    int totalTasks = 0;
    int runningTasks = 0;
    int completedTasks = 0;
    int totalDrivers = 0;
    int queuedDrivers = 0;
    int runningDrivers = 0;
    int blockedDrivers = 0;
    int completedDrivers = 0;
    double cumulativeUserMemory = 0;
    double cumulativeTotalMemory = 0;
    long userMemoryReservation = 0;
    long totalMemoryReservation = 0;
    long totalScheduledTime = 0;
    long totalCpuTime = 0;
    long retriedCpuTime = 0;
    long totalBlockedTime = 0;
    long totalAllocation = 0;
    long rawInputDataSize = 0;
    long rawInputPositions = 0;
    long processedInputDataSize = 0;
    long processedInputPositions = 0;
    long outputDataSize = 0;
    long outputPositions = 0;
    long writtenOutputPositions = 0;
    long writtenOutputLogicalDataSize = 0;
    long writtenOutputPhysicalDataSize = 0;
    long writtenIntermediatePhysicalDataSize = 0;
    ImmutableList.Builder<StageGcStatistics> stageGcStatistics = ImmutableList.builder();
    boolean fullyBlocked = rootStage.isPresent();
    Set<BlockedReason> blockedReasons = new HashSet<>();
    ImmutableList.Builder<OperatorStats> operatorStatsSummary = ImmutableList.builder();
    boolean completeInfo = true;
    RuntimeStats mergedRuntimeStats = RuntimeStats.copyOf(runtimeStats);
    for (StageInfo stageInfo : getAllStages(rootStage)) {
        StageExecutionStats stageExecutionStats = stageInfo.getLatestAttemptExecutionInfo().getStats();
        totalTasks += stageExecutionStats.getTotalTasks();
        runningTasks += stageExecutionStats.getRunningTasks();
        completedTasks += stageExecutionStats.getCompletedTasks();
        totalDrivers += stageExecutionStats.getTotalDrivers();
        queuedDrivers += stageExecutionStats.getQueuedDrivers();
        runningDrivers += stageExecutionStats.getRunningDrivers();
        blockedDrivers += stageExecutionStats.getBlockedDrivers();
        completedDrivers += stageExecutionStats.getCompletedDrivers();
        cumulativeUserMemory += stageExecutionStats.getCumulativeUserMemory();
        cumulativeTotalMemory += stageExecutionStats.getCumulativeTotalMemory();
        userMemoryReservation += stageExecutionStats.getUserMemoryReservation().toBytes();
        totalMemoryReservation += stageExecutionStats.getTotalMemoryReservation().toBytes();
        totalScheduledTime += stageExecutionStats.getTotalScheduledTime().roundTo(MILLISECONDS);
        totalCpuTime += stageExecutionStats.getTotalCpuTime().roundTo(MILLISECONDS);
        retriedCpuTime += computeRetriedCpuTime(stageInfo);
        totalBlockedTime += stageExecutionStats.getTotalBlockedTime().roundTo(MILLISECONDS);
        if (!stageInfo.getLatestAttemptExecutionInfo().getState().isDone()) {
            fullyBlocked &= stageExecutionStats.isFullyBlocked();
            blockedReasons.addAll(stageExecutionStats.getBlockedReasons());
        }
        totalAllocation += stageExecutionStats.getTotalAllocation().toBytes();
        if (stageInfo.getPlan().isPresent()) {
            PlanFragment plan = stageInfo.getPlan().get();
            if (!plan.getTableScanSchedulingOrder().isEmpty()) {
                rawInputDataSize += stageExecutionStats.getRawInputDataSize().toBytes();
                rawInputPositions += stageExecutionStats.getRawInputPositions();
                processedInputDataSize += stageExecutionStats.getProcessedInputDataSize().toBytes();
                processedInputPositions += stageExecutionStats.getProcessedInputPositions();
            }
            if (plan.isOutputTableWriterFragment()) {
                writtenOutputPositions += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(OperatorStats::getInputPositions).sum();
                writtenOutputLogicalDataSize += stageExecutionStats.getOperatorSummaries().stream().filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())).mapToLong(stats -> stats.getInputDataSize().toBytes()).sum();
                writtenOutputPhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
            } else {
                writtenIntermediatePhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes();
            }
        }
        stageGcStatistics.add(stageExecutionStats.getGcInfo());
        completeInfo = completeInfo && stageInfo.isFinalStageInfo();
        operatorStatsSummary.addAll(stageExecutionStats.getOperatorSummaries());
        // We prepend each metric name with the stage id to avoid merging metrics across stages.
        int stageId = stageInfo.getStageId().getId();
        stageExecutionStats.getRuntimeStats().getMetrics().forEach((name, metric) -> {
            String metricName = String.format("S%d-%s", stageId, name);
            mergedRuntimeStats.mergeMetric(metricName, metric);
        });
    }
    if (rootStage.isPresent()) {
        StageExecutionStats outputStageStats = rootStage.get().getLatestAttemptExecutionInfo().getStats();
        outputDataSize += outputStageStats.getOutputDataSize().toBytes();
        outputPositions += outputStageStats.getOutputPositions();
    }
    return new QueryStats(queryStateTimer.getCreateTime(), queryStateTimer.getExecutionStartTime().orElse(null), queryStateTimer.getLastHeartbeat(), queryStateTimer.getEndTime().orElse(null), queryStateTimer.getElapsedTime(), queryStateTimer.getWaitingForPrerequisitesTime(), queryStateTimer.getQueuedTime(), queryStateTimer.getResourceWaitingTime(), queryStateTimer.getSemanticAnalyzingTime(), queryStateTimer.getColumnAccessPermissionCheckingTime(), queryStateTimer.getDispatchingTime(), queryStateTimer.getExecutionTime(), queryStateTimer.getAnalysisTime(), queryStateTimer.getPlanningTime(), queryStateTimer.getFinishingTime(), totalTasks, runningTasks, peakRunningTasks, completedTasks, totalDrivers, queuedDrivers, runningDrivers, blockedDrivers, completedDrivers, cumulativeUserMemory, cumulativeTotalMemory, succinctBytes(userMemoryReservation), succinctBytes(totalMemoryReservation), peakUserMemoryReservation, peakTotalMemoryReservation, peakTaskUserMemory, peakTaskTotalMemory, peakNodeTotalMemory, isScheduled(rootStage), succinctDuration(totalScheduledTime, MILLISECONDS), succinctDuration(totalCpuTime, MILLISECONDS), succinctDuration(retriedCpuTime, MILLISECONDS), succinctDuration(totalBlockedTime, MILLISECONDS), fullyBlocked, blockedReasons, succinctBytes(totalAllocation), succinctBytes(rawInputDataSize), rawInputPositions, succinctBytes(processedInputDataSize), processedInputPositions, succinctBytes(outputDataSize), outputPositions, writtenOutputPositions, succinctBytes(writtenOutputLogicalDataSize), succinctBytes(writtenOutputPhysicalDataSize), succinctBytes(writtenIntermediatePhysicalDataSize), stageGcStatistics.build(), operatorStatsSummary.build(), mergedRuntimeStats);
}

Example 23 with PlanFragment

use of com.facebook.presto.sql.planner.PlanFragment in project presto by prestodb.

the class MockRemoteTaskFactory method createTableScanTask.

public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, List<Split> splits, NodeTaskMap.NodeStatsTracker nodeStatsTracker) {
    VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "column", VARCHAR);
    PlanNodeId sourceId = new PlanNodeId("sourceId");
    PlanFragment testFragment = new PlanFragment(new PlanFragmentId(0), new TableScanNode(Optional.empty(), sourceId, new TableHandle(new ConnectorId("test"), new TestingTableHandle(), TestingTransactionHandle.create(), Optional.of(TestingHandle.INSTANCE)), ImmutableList.of(variable), ImmutableMap.of(variable, new TestingColumnHandle("column")), TupleDomain.all(), TupleDomain.all()), ImmutableSet.of(variable), SOURCE_DISTRIBUTION, ImmutableList.of(sourceId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(variable)), StageExecutionDescriptor.ungroupedExecution(), false, StatsAndCosts.empty(), Optional.empty());
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    for (Split sourceSplit : splits) {
        initialSplits.put(sourceId, sourceSplit);
    }
    return createRemoteTask(TEST_SESSION, taskId, newNode, testFragment, initialSplits.build(), createInitialEmptyOutputBuffers(BROADCAST), nodeStatsTracker, true, new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()));
}

Example 24 with PlanFragment

use of com.facebook.presto.sql.planner.PlanFragment in project presto by prestodb.

the class PhasedExecutionSchedule method extractPhases.

@VisibleForTesting
static List<Set<PlanFragmentId>> extractPhases(Collection<PlanFragment> fragments) {
    // Build a graph where the plan fragments are vertexes and the edges represent
    // a before -> after relationship.  For example, a join hash build has an edge
    // to the join probe.
    DirectedGraph<PlanFragmentId, DefaultEdge> graph = new DefaultDirectedGraph<>(DefaultEdge.class);
    fragments.forEach(fragment -> graph.addVertex(fragment.getId()));
    Visitor visitor = new Visitor(fragments, graph);
    for (PlanFragment fragment : fragments) {
        visitor.processFragment(fragment.getId());
    }
    // Computes all the strongly connected components of the directed graph.
    // These are the "phases" which hold the set of fragments that must be started
    // at the same time to avoid deadlock.
    List<Set<PlanFragmentId>> components = new StrongConnectivityInspector<>(graph).stronglyConnectedSets();
    Map<PlanFragmentId, Set<PlanFragmentId>> componentMembership = new HashMap<>();
    for (Set<PlanFragmentId> component : components) {
        for (PlanFragmentId planFragmentId : component) {
            componentMembership.put(planFragmentId, component);
        }
    }
    // build graph of components (phases)
    DirectedGraph<Set<PlanFragmentId>, DefaultEdge> componentGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
    components.forEach(componentGraph::addVertex);
    for (DefaultEdge edge : graph.edgeSet()) {
        PlanFragmentId source = graph.getEdgeSource(edge);
        PlanFragmentId target = graph.getEdgeTarget(edge);
        Set<PlanFragmentId> from = componentMembership.get(source);
        Set<PlanFragmentId> to = componentMembership.get(target);
        if (!from.equals(to)) {
            // the topological order iterator below doesn't include vertices that have self-edges, so don't add them
            componentGraph.addEdge(from, to);
        }
    }
    List<Set<PlanFragmentId>> schedulePhases = ImmutableList.copyOf(new TopologicalOrderIterator<>(componentGraph));
    return schedulePhases;
}

Example 25 with PlanFragment

use of com.facebook.presto.sql.planner.PlanFragment in project presto by prestodb.

the class SqlQueryScheduler method createStageExecutions.

private List<SectionExecution> createStageExecutions(List<StreamingPlanSection> sections) {
    ImmutableList.Builder<SectionExecution> result = ImmutableList.builder();
    for (StreamingPlanSection section : sections) {
        StageId sectionId = getStageId(section.getPlan().getFragment().getId());
        List<SectionExecution> attempts = sectionExecutions.computeIfAbsent(sectionId, (ignored) -> new CopyOnWriteArrayList<>());
        // sectionExecutions only get created when they are about to be scheduled, so there should
        // never be a non-failed SectionExecution for a section that's ready for execution
        verify(attempts.isEmpty() || getLast(attempts).isFailed(), "Non-failed sectionExecutions already exists");
        PlanFragment sectionRootFragment = section.getPlan().getFragment();
        Optional<int[]> bucketToPartition;
        OutputBuffers outputBuffers;
        ExchangeLocationsConsumer locationsConsumer;
        if (isRootFragment(sectionRootFragment)) {
            bucketToPartition = Optional.of(new int[1]);
            outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
            OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
        } else {
            bucketToPartition = Optional.empty();
            outputBuffers = createDiscardingOutputBuffers();
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
            };
        }
        int attemptId = attempts.size();
        SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, section, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
        addStateChangeListeners(sectionExecution);
        attempts.add(sectionExecution);
        result.add(sectionExecution);
    }
    return result.build();
}