Examples with OutputBufferId com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId used on opensource projects

Example 1 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class LegacySqlQueryScheduler method updateStageExecutions.

/**
 * Utility function that rebuild a StreamingPlanSection, re-create stageExecutionAndScheduler for each of its stage, and finally update the stageExecutions map.
 */
private void updateStageExecutions(StreamingPlanSection section, Map<PlanFragment, PlanFragment> oldToNewFragment) {
    StreamingPlanSection newSection = new StreamingPlanSection(rewriteStreamingSubPlan(section.getPlan(), oldToNewFragment), section.getChildren());
    PlanFragment sectionRootFragment = newSection.getPlan().getFragment();
    Optional<int[]> bucketToPartition;
    OutputBuffers outputBuffers;
    ExchangeLocationsConsumer locationsConsumer;
    if (isRootFragment(sectionRootFragment)) {
        bucketToPartition = Optional.of(new int[1]);
        outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
        OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
    } else {
        bucketToPartition = Optional.empty();
        outputBuffers = createDiscardingOutputBuffers();
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
        };
    }
    SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, newSection, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, 0);
    addStateChangeListeners(sectionExecution);
    Map<StageId, StageExecutionAndScheduler> updatedStageExecutions = sectionExecution.getSectionStages().stream().collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity()));
    synchronized (this) {
        stageExecutions.putAll(updatedStageExecutions);
    }
}

Example 2 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class ScaledOutputBufferManager method addOutputBuffers.

@SuppressWarnings("ObjectEquality")
@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // so ignore the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, newBuffer.getId());
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}

Example 3 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    }, directExecutor());
    // For hard timeout, add an additional time to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}

Example 4 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class BroadcastOutputBufferManager method addOutputBuffers.

@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}

Example 5 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class ArbitraryOutputBuffer method setOutputBuffers.

@Override
public void setOutputBuffers(OutputBuffers newOutputBuffers) {
    checkState(!Thread.holdsLock(this), "Can not set output buffers while holding a lock on this");
    requireNonNull(newOutputBuffers, "newOutputBuffers is null");
    synchronized (this) {
        // ignore buffers added after query finishes, which can happen when a query is canceled
        // also ignore old versions, which is normal
        BufferState state = this.state.get();
        if (state.isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) {
            return;
        }
        // verify this is valid state change
        outputBuffers.checkValidTransition(newOutputBuffers);
        outputBuffers = newOutputBuffers;
        // add the new buffers
        for (OutputBufferId outputBufferId : outputBuffers.getBuffers().keySet()) {
            getBuffer(outputBufferId);
        }
        // Reset resume from position
        nextClientBufferIndex.set(0);
        // update state if no more buffers is set
        if (outputBuffers.isNoMoreBufferIds()) {
            this.state.compareAndSet(OPEN, NO_MORE_BUFFERS);
            this.state.compareAndSet(NO_MORE_PAGES, FLUSHING);
        }
    }
    if (!state.get().canAddBuffers()) {
        noMoreBuffers();
    }
    checkFlushComplete();
}