use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.
the class BroadcastOutputBufferManager method addOutputBuffers.
@Override
public void addOutputBuffers(List<OutputBufferId> newBuffers, boolean noMoreBuffers) {
OutputBuffers newOutputBuffers;
synchronized (this) {
if (outputBuffers.isNoMoreBufferIds()) {
// the new buffers
return;
}
OutputBuffers originalOutputBuffers = outputBuffers;
// Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition
for (OutputBufferId newBuffer : newBuffers) {
outputBuffers = outputBuffers.withBuffer(newBuffer, BROADCAST_PARTITION_ID);
}
if (noMoreBuffers) {
outputBuffers = outputBuffers.withNoMoreBufferIds();
}
// don't update if nothing changed
if (outputBuffers == originalOutputBuffers) {
return;
}
newOutputBuffers = this.outputBuffers;
}
outputBufferTarget.accept(newOutputBuffers);
}
use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.
the class TaskResource method getResults.
@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) throws InterruptedException {
requireNonNull(taskId, "taskId is null");
requireNonNull(bufferId, "bufferId is null");
long start = System.nanoTime();
ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
List<SerializedPage> serializedPages = result.getSerializedPages();
GenericEntity<?> entity = null;
Status status;
if (serializedPages.isEmpty()) {
status = Status.NO_CONTENT;
} else {
entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
}.getType());
status = Status.OK;
}
return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
});
// For hard timeout, add an additional 5 seconds to max wait for thread scheduling contention and GC
Duration timeout = new Duration(waitTime.toMillis() + 5000, MILLISECONDS);
bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}
use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.
the class ArbitraryOutputBuffer method setOutputBuffers.
@Override
public void setOutputBuffers(OutputBuffers newOutputBuffers) {
checkState(!Thread.holdsLock(this), "Can not set output buffers while holding a lock on this");
requireNonNull(newOutputBuffers, "newOutputBuffers is null");
synchronized (this) {
// ignore buffers added after query finishes, which can happen when a query is canceled
// also ignore old versions, which is normal
BufferState state = this.state.get();
if (state.isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) {
return;
}
// verify this is valid state change
outputBuffers.checkValidTransition(newOutputBuffers);
outputBuffers = newOutputBuffers;
// add the new buffers
for (OutputBufferId outputBufferId : outputBuffers.getBuffers().keySet()) {
getBuffer(outputBufferId);
}
// update state if no more buffers is set
if (outputBuffers.isNoMoreBufferIds()) {
this.state.compareAndSet(OPEN, NO_MORE_BUFFERS);
this.state.compareAndSet(NO_MORE_PAGES, FLUSHING);
}
}
if (!state.get().canAddBuffers()) {
noMoreBuffers();
}
checkFlushComplete();
}
use of com.facebook.presto.OutputBuffers.OutputBufferId in project presto by prestodb.
the class TestBroadcastOutputBufferManager method test.
@Test
public void test() throws Exception {
AtomicReference<OutputBuffers> outputBufferTarget = new AtomicReference<>();
BroadcastOutputBufferManager hashOutputBufferManager = new BroadcastOutputBufferManager(outputBufferTarget::set);
assertEquals(outputBufferTarget.get(), createInitialEmptyOutputBuffers(BROADCAST));
hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(0)), false);
OutputBuffers expectedOutputBuffers = createInitialEmptyOutputBuffers(BROADCAST).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID);
assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(1), new OutputBufferId(2)), false);
expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(1), BROADCAST_PARTITION_ID);
expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(2), BROADCAST_PARTITION_ID);
assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
// set no more buffers
hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(3)), true);
expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(3), BROADCAST_PARTITION_ID);
expectedOutputBuffers = expectedOutputBuffers.withNoMoreBufferIds();
assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
// try to add another buffer, which should not result in an error
// and output buffers should not change
hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(5)), false);
assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
// try to set no more buffers again, which should not result in an error
// and output buffers should not change
hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(6)), true);
assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
}
Aggregations