Examples with DistributedExecutionPlanner io.prestosql.sql.planner.DistributedExecutionPlanner used on opensource projects

Example 1 with DistributedExecutionPlanner

use of io.prestosql.sql.planner.DistributedExecutionPlanner in project hetu-core by openlookeng.

the class SqlQueryExecution method createResumeScheduler.

private SqlQueryScheduler createResumeScheduler(PlanRoot plan, OutputBuffers rootOutputBuffers) {
    String resumeMessage = "Query encountered failures. Recovering using the distributed-snapshot feature.";
    warningCollector.add(new PrestoWarning(StandardWarningCode.SNAPSHOT_RECOVERY, resumeMessage));
    // Check if there is a snapshot we can restore to, or restart from beginning,
    // and update marker split sources so they know where to resume from.
    // This MUST be done BEFORE creating the new scheduler, because it resets the snapshotManager internal states.
    OptionalLong snapshotId = snapshotManager.getResumeSnapshotId();
    MarkerAnnouncer announcer = splitManager.getMarkerAnnouncer(stateMachine.getSession());
    announcer.resumeSnapshot(snapshotId.orElse(0));
    // Clear any temporary content that's not part of the snapshot
    resetOutputData(plan, snapshotId);
    // Create a new scheduler, to schedule new stages and tasks
    DistributedExecutionPlanner distributedExecutionPlanner = new DistributedExecutionPlanner(splitManager, metadata);
    StageExecutionPlan executionPlan = distributedExecutionPlanner.plan(plan.getRoot(), stateMachine.getSession(), RESUME, snapshotId.isPresent() ? snapshotId.getAsLong() : null, announcer.currentSnapshotId());
    // build the stage execution objects (this doesn't schedule execution)
    return createSqlQueryScheduler(stateMachine, locationFactory, executionPlan, nodePartitioningManager, nodeScheduler, remoteTaskFactory, stateMachine.getSession(), plan.isSummarizeTaskInfos(), scheduleSplitBatchSize, queryExecutor, schedulerExecutor, failureDetector, rootOutputBuffers, nodeTaskMap, executionPolicy, schedulerStats, dynamicFilterService, heuristicIndexerManager, snapshotManager, // Require same number of tasks to be scheduled, but do not require it if starting from beginning
    snapshotId.isPresent() ? queryScheduler.get().getStageTaskCounts() : null);
}

Example 2 with DistributedExecutionPlanner

use of io.prestosql.sql.planner.DistributedExecutionPlanner in project hetu-core by openlookeng.

the class SqlQueryExecution method planDistribution.

private void planDistribution(PlanRoot plan) {
    // time distribution planning
    stateMachine.beginDistributedPlanning();
    // plan the execution on the active nodes
    DistributedExecutionPlanner distributedPlanner = new DistributedExecutionPlanner(splitManager, metadata);
    StageExecutionPlan outputStageExecutionPlan;
    Session session = stateMachine.getSession();
    if (SystemSessionProperties.isSnapshotEnabled(session)) {
        // Snapshot: need to plan different when snapshot is enabled.
        // See the "plan" method for difference between the different modes.
        MarkerAnnouncer announcer = splitManager.getMarkerAnnouncer(session);
        announcer.setSnapshotManager(snapshotManager);
        outputStageExecutionPlan = distributedPlanner.plan(plan.getRoot(), session, SNAPSHOT, null, announcer.currentSnapshotId());
    } else {
        outputStageExecutionPlan = distributedPlanner.plan(plan.getRoot(), session, NORMAL, null, 0);
    }
    stateMachine.endDistributedPlanning();
    // ensure split sources are closed
    stateMachine.addStateChangeListener(state -> {
        if (state.isDone()) {
            closeSplitSources(outputStageExecutionPlan);
        }
    });
    // if query was canceled, skip creating scheduler
    if (stateMachine.isDone()) {
        return;
    }
    // record output field
    stateMachine.setColumns(outputStageExecutionPlan.getFieldNames(), outputStageExecutionPlan.getFragment().getTypes());
    PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle();
    OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle).withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    // build the stage execution objects (this doesn't schedule execution)
    SqlQueryScheduler scheduler = createSqlQueryScheduler(stateMachine, locationFactory, outputStageExecutionPlan, nodePartitioningManager, nodeScheduler, remoteTaskFactory, stateMachine.getSession(), plan.isSummarizeTaskInfos(), scheduleSplitBatchSize, queryExecutor, schedulerExecutor, failureDetector, rootOutputBuffers, nodeTaskMap, executionPolicy, schedulerStats, dynamicFilterService, heuristicIndexerManager, snapshotManager, null);
    queryScheduler.set(scheduler);
    // directly since the callback may have already fired
    if (stateMachine.isDone()) {
        scheduler.abort();
        queryScheduler.set(null);
    }
}