Search in sources :

Example 1 with OutputBufferId

use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.

the class BroadcastOutputBufferManager method addOutputBuffers.

@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
    OutputBuffers newOutputBuffers;
    synchronized (this) {
        if (outputBuffers.isNoMoreBufferIds()) {
            // the new buffers
            return;
        }
        OutputBuffers originalOutputBuffers = outputBuffers;
        // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
        for (OutputBufferId newBuffer : newBuffers) {
            outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
        }
        if (noMoreBuffers) {
            outputBuffers = outputBuffers.withNoMoreBufferIds();
        }
        // don't update if nothing changed
        if (outputBuffers == originalOutputBuffers) {
            return;
        }
        newOutputBuffers = this.outputBuffers;
    }
    outputBufferTarget.accept(newOutputBuffers);
}
Also used : OutputBuffers(com.facebook.presto.OutputBuffers) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers) OutputBufferId(com.facebook.presto.OutputBuffers.OutputBufferId)

Example 2 with OutputBufferId

use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    });
    // For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}
Also used : TaskStatus(com.facebook.presto.execution.TaskStatus) Status(javax.ws.rs.core.Response.Status) Page(com.facebook.presto.spi.Page) Produces(javax.ws.rs.Produces) Iterables.transform(com.google.common.collect.Iterables.transform) TaskStatus(com.facebook.presto.execution.TaskStatus) Path(javax.ws.rs.Path) AsyncResponseHandler.bindAsyncResponse(io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse) TaskState(com.facebook.presto.execution.TaskState) Duration(io.airlift.units.Duration) OutputBufferId(com.facebook.presto.OutputBuffers.OutputBufferId) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) DefaultValue(javax.ws.rs.DefaultValue) HeaderParam(javax.ws.rs.HeaderParam) BufferResult(com.facebook.presto.execution.buffer.BufferResult) DELETE(javax.ws.rs.DELETE) PRESTO_PAGE_TOKEN(com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_TOKEN) Context(javax.ws.rs.core.Context) AsyncResponse(javax.ws.rs.container.AsyncResponse) GenericEntity(javax.ws.rs.core.GenericEntity) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Suspended(javax.ws.rs.container.Suspended) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) DataSize(io.airlift.units.DataSize) List(java.util.List) Response(javax.ws.rs.core.Response) CompletionCallback(javax.ws.rs.container.CompletionCallback) PRESTO_BUFFER_COMPLETE(com.facebook.presto.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) UriInfo(javax.ws.rs.core.UriInfo) PRESTO_CURRENT_STATE(com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE) Nested(org.weakref.jmx.Nested) PathParam(javax.ws.rs.PathParam) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) GET(javax.ws.rs.GET) TypeToken(com.google.common.reflect.TypeToken) Inject(javax.inject.Inject) PRESTO_TASK_INSTANCE_ID(com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID) ImmutableList(com.google.common.collect.ImmutableList) Managed(org.weakref.jmx.Managed) PRESTO_MAX_SIZE(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_SIZE) TaskManager(com.facebook.presto.execution.TaskManager) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Objects.requireNonNull(java.util.Objects.requireNonNull) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MoreFutures.addTimeout(io.airlift.concurrent.MoreFutures.addTimeout) PRESTO_PAGE_NEXT_TOKEN(com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_NEXT_TOKEN) SessionPropertyManager(com.facebook.presto.metadata.SessionPropertyManager) TimeStat(io.airlift.stats.TimeStat) Status(javax.ws.rs.core.Response.Status) POST(javax.ws.rs.POST) Executor(java.util.concurrent.Executor) Session(com.facebook.presto.Session) PRESTO_MAX_WAIT(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT) Futures(com.google.common.util.concurrent.Futures) PRESTO_PAGES(com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES) TaskId(com.facebook.presto.execution.TaskId) TaskInfo(com.facebook.presto.execution.TaskInfo) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Duration(io.airlift.units.Duration) AsyncResponseHandler.bindAsyncResponse(io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse) AsyncResponse(javax.ws.rs.container.AsyncResponse) Response(javax.ws.rs.core.Response) BufferResult(com.facebook.presto.execution.buffer.BufferResult) SerializedPage(com.facebook.presto.execution.buffer.SerializedPage) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET)

Example 3 with OutputBufferId

use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.

the class ArbitraryOutputBuffer method setOutputBuffers.

@Override
public void setOutputBuffers(OutputBuffers newOutputBuffers) {
    checkState(!Thread.holdsLock(this), "Can not set output buffers while holding a lock on this");
    requireNonNull(newOutputBuffers, "newOutputBuffers is null");
    synchronized (this) {
        // ignore buffers added after query finishes, which can happen when a query is canceled
        // also ignore old versions, which is normal
        BufferState state = this.state.get();
        if (state.isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) {
            return;
        }
        // verify this is valid state change
        outputBuffers.checkValidTransition(newOutputBuffers);
        outputBuffers = newOutputBuffers;
        // add the new buffers
        for (OutputBufferId outputBufferId : outputBuffers.getBuffers().keySet()) {
            getBuffer(outputBufferId);
        }
        // update state if no more buffers is set
        if (outputBuffers.isNoMoreBufferIds()) {
            this.state.compareAndSet(OPEN, NO_MORE_BUFFERS);
            this.state.compareAndSet(NO_MORE_PAGES, FLUSHING);
        }
    }
    if (!state.get().canAddBuffers()) {
        noMoreBuffers();
    }
    checkFlushComplete();
}
Also used : OutputBufferId(com.facebook.presto.OutputBuffers.OutputBufferId)

Example 4 with OutputBufferId

use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TestBroadcastOutputBufferManager method test.

@Test
public void test() throws Exception {
    AtomicReference<OutputBuffers> outputBufferTarget = new AtomicReference<>();
    BroadcastOutputBufferManager hashOutputBufferManager = new BroadcastOutputBufferManager(outputBufferTarget::set);
    assertEquals(outputBufferTarget.get(), createInitialEmptyOutputBuffers(BROADCAST));
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(0)), false);
    OutputBuffers expectedOutputBuffers = createInitialEmptyOutputBuffers(BROADCAST).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(1), new OutputBufferId(2)), false);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(1), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(2), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // set no more buffers
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(3)), true);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(3), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withNoMoreBufferIds();
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to add another buffer, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(5)), false);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to set no more buffers again, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(6)), true);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
}
Also used : OutputBuffers(com.facebook.presto.OutputBuffers) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers) OutputBufferId(com.facebook.presto.OutputBuffers.OutputBufferId) AtomicReference(java.util.concurrent.atomic.AtomicReference) Test(org.testng.annotations.Test)

Aggregations

OutputBufferId (com.facebook.presto.OutputBuffers.OutputBufferId)4 OutputBuffers (com.facebook.presto.OutputBuffers)2 OutputBuffers.createInitialEmptyOutputBuffers (com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers)2 PRESTO_PAGES (com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES)1 Session (com.facebook.presto.Session)1 PRESTO_BUFFER_COMPLETE (com.facebook.presto.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE)1 PRESTO_CURRENT_STATE (com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE)1 PRESTO_MAX_SIZE (com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_SIZE)1 PRESTO_MAX_WAIT (com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT)1 PRESTO_PAGE_NEXT_TOKEN (com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_NEXT_TOKEN)1 PRESTO_PAGE_TOKEN (com.facebook.presto.client.PrestoHeaders.PRESTO_PAGE_TOKEN)1 PRESTO_TASK_INSTANCE_ID (com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID)1 TaskId (com.facebook.presto.execution.TaskId)1 TaskInfo (com.facebook.presto.execution.TaskInfo)1 TaskManager (com.facebook.presto.execution.TaskManager)1 TaskState (com.facebook.presto.execution.TaskState)1 TaskStatus (com.facebook.presto.execution.TaskStatus)1 BufferResult (com.facebook.presto.execution.buffer.BufferResult)1 SerializedPage (com.facebook.presto.execution.buffer.SerializedPage)1 SessionPropertyManager (com.facebook.presto.metadata.SessionPropertyManager)1