Examples with OutputBuffers com.facebook.presto.OutputBuffers used on opensource projects

Example 1 with OutputBuffers

use of com.facebook.presto.OutputBuffers in project presto by prestodb.

the class BroadcastOutputBufferManager method addOutputBuffers.

@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}

Example 2 with OutputBuffers

use of com.facebook.presto.OutputBuffers in project presto by prestodb.

the class SqlStageExecution method scheduleTask.

private synchronized RemoteTask scheduleTask(Node node, TaskId taskId, Multimap<PlanNodeId, Split> sourceSplits) {
    checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId);
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    initialSplits.putAll(sourceSplits);
    for (Entry<PlanNodeId, URI> entry : exchangeLocations.entries()) {
        initialSplits.put(entry.getKey(), createRemoteSplitFor(taskId, entry.getValue()));
    }
    OutputBuffers outputBuffers = this.outputBuffers.get();
    checkState(outputBuffers != null, "Initial output buffers must be set before a task can be scheduled");
    RemoteTask task = remoteTaskFactory.createRemoteTask(stateMachine.getSession(), taskId, node, stateMachine.getFragment(), initialSplits.build(), outputBuffers, nodeTaskMap.createPartitionedSplitCountTracker(node, taskId), summarizeTaskInfo);
    completeSources.forEach(task::noMoreSplits);
    allTasks.add(taskId);
    tasks.computeIfAbsent(node, key -> newConcurrentHashSet()).add(task);
    nodeTaskMap.addTask(node, task);
    task.addStateChangeListener(new StageTaskListener());
    if (!stateMachine.getState().isDone()) {
        task.start();
    } else {
        // stage finished while we were scheduling this task
        task.abort();
    }
    return task;
}

Example 3 with OutputBuffers

use of com.facebook.presto.OutputBuffers in project presto by prestodb.

the class SqlQueryExecution method planDistribution.

private void planDistribution(PlanRoot plan) {
    // time distribution planning
    long distributedPlanningStart = System.nanoTime();
    // plan the execution on the active nodes
    DistributedExecutionPlanner distributedPlanner = new DistributedExecutionPlanner(splitManager);
    StageExecutionPlan outputStageExecutionPlan = distributedPlanner.plan(plan.getRoot(), stateMachine.getSession());
    stateMachine.recordDistributedPlanningTime(distributedPlanningStart);
    // ensure split sources are closed
    stateMachine.addStateChangeListener(state -> {
        if (state.isDone()) {
            closeSplitSources(outputStageExecutionPlan);
        }
    });
    // if query was canceled, skip creating scheduler
    if (stateMachine.isDone()) {
        return;
    }
    // record field names
    stateMachine.setOutputFieldNames(outputStageExecutionPlan.getFieldNames());
    PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle();
    OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle).withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    // build the stage execution objects (this doesn't schedule execution)
    SqlQueryScheduler scheduler = new SqlQueryScheduler(stateMachine, locationFactory, outputStageExecutionPlan, nodePartitioningManager, nodeScheduler, remoteTaskFactory, stateMachine.getSession(), plan.isSummarizeTaskInfos(), scheduleSplitBatchSize, queryExecutor, rootOutputBuffers, nodeTaskMap, executionPolicy, schedulerStats);
    queryScheduler.set(scheduler);
    // directly since the callback may have already fired
    if (stateMachine.isDone()) {
        scheduler.abort();
        queryScheduler.set(null);
    }
}

Example 4 with OutputBuffers

use of com.facebook.presto.OutputBuffers in project presto by prestodb.

the class TestArbitraryOutputBuffer method testAddQueueAfterNoMoreQueues.

@Test
public void testAddQueueAfterNoMoreQueues() throws Exception {
    ArbitraryOutputBuffer buffer = createArbitraryBuffer(createInitialEmptyOutputBuffers(ARBITRARY), sizeOfPages(10));
    assertFalse(buffer.isFinished());
    // tell buffer no more queues will be added
    buffer.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY).withNoMoreBufferIds());
    assertFalse(buffer.isFinished());
    // set no more queues a second time to assure that we don't get an exception or such
    buffer.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY).withNoMoreBufferIds());
    assertFalse(buffer.isFinished());
    // set no more queues a third time to assure that we don't get an exception or such
    buffer.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY).withNoMoreBufferIds());
    assertFalse(buffer.isFinished());
    try {
        OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(ARBITRARY).withBuffer(FIRST, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
        buffer.setOutputBuffers(outputBuffers);
        fail("Expected IllegalStateException from addQueue after noMoreQueues has been called");
    } catch (IllegalArgumentException ignored) {
    }
}

Example 5 with OutputBuffers

use of com.facebook.presto.OutputBuffers in project presto by prestodb.

the class TestArbitraryOutputBuffer method testAbort.

@Test
public void testAbort() throws Exception {
    ArbitraryOutputBuffer buffer = createArbitraryBuffer(createInitialEmptyOutputBuffers(ARBITRARY), sizeOfPages(10));
    // fill the buffer
    for (int i = 0; i < 10; i++) {
        addPage(buffer, createPage(i));
    }
    buffer.setNoMorePages();
    // add one output buffer
    OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(ARBITRARY).withBuffer(FIRST, 0);
    buffer.setOutputBuffers(outputBuffers);
    // read a page from the first buffer
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 0, sizeOfPages(1), NO_WAIT), bufferResult(0, createPage(0)));
    // abort buffer, and verify page cannot be acknowledged
    buffer.abort(FIRST);
    assertQueueClosed(buffer, 9, FIRST, 0);
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 1, sizeOfPages(1), NO_WAIT), emptyResults(TASK_INSTANCE_ID, 0, true));
    outputBuffers = outputBuffers.withBuffer(SECOND, 0).withNoMoreBufferIds();
    buffer.setOutputBuffers(outputBuffers);
    // first page is lost because the first buffer was aborted
    assertBufferResultEquals(TYPES, getBufferResult(buffer, SECOND, 0, sizeOfPages(1), NO_WAIT), bufferResult(0, createPage(1)));
    buffer.abort(SECOND);
    assertQueueClosed(buffer, 0, SECOND, 0);
    assertFinished(buffer);
    assertBufferResultEquals(TYPES, getBufferResult(buffer, SECOND, 1, sizeOfPages(1), NO_WAIT), emptyResults(TASK_INSTANCE_ID, 0, true));
}