Examples with OutputBuffers io.prestosql.execution.buffer.OutputBuffers used on opensource projects

Example 1 with OutputBuffers

use of io.prestosql.execution.buffer.OutputBuffers in project hetu-core by openlookeng.

the class SqlStageExecution method scheduleTask.

private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, String instanceId, Multimap<PlanNodeId, Split> sourceSplits, OptionalInt totalPartitions) {
    checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId);
    if (SystemSessionProperties.isSnapshotEnabled(stateMachine.getSession())) {
        // Snapshot: inform snapshot manager so it knows about all tasks,
        // and can determine if a snapshot is complete for all tasks.
        snapshotManager.addNewTask(taskId);
    }
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    initialSplits.putAll(sourceSplits);
    sourceTasks.forEach((planNodeId, task) -> {
        if (task.getTaskStatus().getState() != TaskState.FINISHED) {
            initialSplits.put(planNodeId, newConnectSplit(taskId, task));
        }
    });
    OutputBuffers localOutputBuffers = this.outputBuffers.get();
    checkState(localOutputBuffers != null, "Initial output buffers must be set before a task can be scheduled");
    RemoteTask task = remoteTaskFactory.createRemoteTask(stateMachine.getSession(), taskId, instanceId, node, stateMachine.getFragment(), initialSplits.build(), totalPartitions, localOutputBuffers, nodeTaskMap.createPartitionedSplitCountTracker(node, taskId), summarizeTaskInfo, Optional.ofNullable(parentId), snapshotManager);
    completeSources.forEach(task::noMoreSplits);
    allTasks.add(taskId);
    tasks.computeIfAbsent(node, key -> newConcurrentHashSet()).add(task);
    nodeTaskMap.addTask(node, task);
    task.addStateChangeListener(new StageTaskListener());
    task.addFinalTaskInfoListener(this::updateFinalTaskInfo);
    if (!stateMachine.getState().isDone()) {
        task.start();
    } else {
        // stage finished while we were scheduling this task
        task.abort();
    }
    return task;
}

Example 2 with OutputBuffers

use of io.prestosql.execution.buffer.OutputBuffers in project hetu-core by openlookeng.

the class SqlQueryExecution method planDistribution.

private void planDistribution(PlanRoot plan) {
    // time distribution planning
    stateMachine.beginDistributedPlanning();
    // plan the execution on the active nodes
    DistributedExecutionPlanner distributedPlanner = new DistributedExecutionPlanner(splitManager, metadata);
    StageExecutionPlan outputStageExecutionPlan;
    Session session = stateMachine.getSession();
    if (SystemSessionProperties.isSnapshotEnabled(session)) {
        // Snapshot: need to plan different when snapshot is enabled.
        // See the "plan" method for difference between the different modes.
        MarkerAnnouncer announcer = splitManager.getMarkerAnnouncer(session);
        announcer.setSnapshotManager(snapshotManager);
        outputStageExecutionPlan = distributedPlanner.plan(plan.getRoot(), session, SNAPSHOT, null, announcer.currentSnapshotId());
    } else {
        outputStageExecutionPlan = distributedPlanner.plan(plan.getRoot(), session, NORMAL, null, 0);
    }
    stateMachine.endDistributedPlanning();
    // ensure split sources are closed
    stateMachine.addStateChangeListener(state -> {
        if (state.isDone()) {
            closeSplitSources(outputStageExecutionPlan);
        }
    });
    // if query was canceled, skip creating scheduler
    if (stateMachine.isDone()) {
        return;
    }
    // record output field
    stateMachine.setColumns(outputStageExecutionPlan.getFieldNames(), outputStageExecutionPlan.getFragment().getTypes());
    PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle();
    OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle).withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    // build the stage execution objects (this doesn't schedule execution)
    SqlQueryScheduler scheduler = createSqlQueryScheduler(stateMachine, locationFactory, outputStageExecutionPlan, nodePartitioningManager, nodeScheduler, remoteTaskFactory, stateMachine.getSession(), plan.isSummarizeTaskInfos(), scheduleSplitBatchSize, queryExecutor, schedulerExecutor, failureDetector, rootOutputBuffers, nodeTaskMap, executionPolicy, schedulerStats, dynamicFilterService, heuristicIndexerManager, snapshotManager, null);
    queryScheduler.set(scheduler);
    // directly since the callback may have already fired
    if (stateMachine.isDone()) {
        scheduler.abort();
        queryScheduler.set(null);
    }
}

Example 3 with OutputBuffers

use of io.prestosql.execution.buffer.OutputBuffers in project hetu-core by openlookeng.

the class SqlQueryExecution method resumeQuery.

private void resumeQuery(PlanRoot plan) {
    SqlQueryScheduler oldScheduler = queryScheduler.get();
    try {
        // Wait for previous scheduler to finish.
        // This is important, otherwise the old schedule may close split sources after the new scheduler has started.
        oldScheduler.doneScheduling().get();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    log.debug("Rescheduling query %s from a resumable task failure.", getQueryId());
    PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle();
    OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle).withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    // build the stage execution objects (this doesn't schedule execution)
    SqlQueryScheduler scheduler;
    try {
        scheduler = createResumeScheduler(plan, rootOutputBuffers);
    } catch (PrestoException e) {
        if (e.getErrorCode() == NO_NODES_AVAILABLE.toErrorCode()) {
            // Not enough worker to resume all tasks. Retrying from any saved snapshot likely wont' work either.
            // Clear ongoing and existing snapshots and restart.
            snapshotManager.invalidateAllSnapshots();
            scheduler = createResumeScheduler(plan, rootOutputBuffers);
        } else {
            throw e;
        }
    }
    queryScheduler.set(scheduler);
    log.debug("Restarting query %s from a resumable task failure.", getQueryId());
    scheduler.start();
    stateMachine.transitionToStarting();
}

Example 4 with OutputBuffers

use of io.prestosql.execution.buffer.OutputBuffers in project hetu-core by openlookeng.

the class BroadcastOutputBufferManager method addOutputBuffers.

@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}

Example 5 with OutputBuffers

use of io.prestosql.execution.buffer.OutputBuffers in project hetu-core by openlookeng.

the class ScaledOutputBufferManager method addOutputBuffers.

@SuppressWarnings("ObjectEquality")
@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // so ignore the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, newBuffer.getId());
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}