Search in sources :

Example 31 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class NestedLoopJoinPagesBuilder method restore.

@Override
public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
    NestedLoopJoinPagesBuilderState myState = (NestedLoopJoinPagesBuilderState) state;
    this.pages.clear();
    PagesSerde serde = (PagesSerde) serdeProvider;
    for (Object obj : myState.pages) {
        SerializedPage sp = SerializedPage.restoreSerializedPage(obj);
        this.pages.add(serde.deserialize(sp));
    }
    this.finished = myState.finished;
    this.estimatedSize = myState.estimatedSize;
}
Also used : PagesSerde(io.hetu.core.transport.execution.buffer.PagesSerde) SerializedPage(io.hetu.core.transport.execution.buffer.SerializedPage)

Example 32 with SerializedPage

use of io.hetu.core.transport.execution.buffer.SerializedPage in project hetu-core by openlookeng.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @HeaderParam(PRESTO_TASK_INSTANCE_ID) String taskInstanceId, @Suspended AsyncResponse asyncResponse) {
    SecurityRequireNonNull.requireNonNull(taskId, "taskId is null");
    SecurityRequireNonNull.requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize, taskInstanceId);
    if (bufferResultFuture == null) {
        // Request came from task has been cancelled.
        asyncResponse.resume(Response.status(Status.NO_CONTENT).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, // keep requesting task running, until they are cancelled (to resume)
        false).build());
        return;
    }
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    }, directExecutor());
    // For hard timeout, add an additional time to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}
Also used : TaskStatus(io.prestosql.execution.TaskStatus) Status(javax.ws.rs.core.Response.Status) TaskStatus.initialTaskStatus(io.prestosql.execution.TaskStatus.initialTaskStatus) PRESTO_TASK_INSTANCE_ID(io.prestosql.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID) Produces(javax.ws.rs.Produces) Iterables.transform(com.google.common.collect.Iterables.transform) BufferResult(io.prestosql.execution.buffer.BufferResult) Path(javax.ws.rs.Path) PRESTO_BUFFER_COMPLETE(io.prestosql.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE) Duration(io.airlift.units.Duration) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) Consumes(javax.ws.rs.Consumes) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) HeaderParam(javax.ws.rs.HeaderParam) URI(java.net.URI) DELETE(javax.ws.rs.DELETE) Context(javax.ws.rs.core.Context) TaskStatus(io.prestosql.execution.TaskStatus) SecurityRequireNonNull(io.prestosql.server.security.SecurityRequireNonNull) AsyncResponse(javax.ws.rs.container.AsyncResponse) GenericEntity(javax.ws.rs.core.GenericEntity) TaskInfo(io.prestosql.execution.TaskInfo) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Suspended(javax.ws.rs.container.Suspended) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) DataSize(io.airlift.units.DataSize) List(java.util.List) SerializedPage(io.hetu.core.transport.execution.buffer.SerializedPage) Response(javax.ws.rs.core.Response) CompletionCallback(javax.ws.rs.container.CompletionCallback) SessionPropertyManager(io.prestosql.metadata.SessionPropertyManager) UriInfo(javax.ws.rs.core.UriInfo) AsyncResponseHandler.bindAsyncResponse(io.airlift.jaxrs.AsyncResponseHandler.bindAsyncResponse) PRESTO_PAGE_NEXT_TOKEN(io.prestosql.client.PrestoHeaders.PRESTO_PAGE_NEXT_TOKEN) PRESTO_MAX_SIZE(io.prestosql.client.PrestoHeaders.PRESTO_MAX_SIZE) TaskId(io.prestosql.execution.TaskId) Nested(org.weakref.jmx.Nested) PathParam(javax.ws.rs.PathParam) TaskStats(io.prestosql.operator.TaskStats) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) GET(javax.ws.rs.GET) PRESTO_CURRENT_STATE(io.prestosql.client.PrestoHeaders.PRESTO_CURRENT_STATE) TypeToken(com.google.common.reflect.TypeToken) PRESTO_MAX_WAIT(io.prestosql.client.PrestoHeaders.PRESTO_MAX_WAIT) Inject(javax.inject.Inject) PRESTO_PAGE_TOKEN(io.prestosql.client.PrestoHeaders.PRESTO_PAGE_TOKEN) ImmutableList(com.google.common.collect.ImmutableList) Managed(org.weakref.jmx.Managed) OutputBufferId(io.prestosql.execution.buffer.OutputBuffers.OutputBufferId) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MoreFutures.addTimeout(io.airlift.concurrent.MoreFutures.addTimeout) TimeStat(io.airlift.stats.TimeStat) TaskManager(io.prestosql.execution.TaskManager) Status(javax.ws.rs.core.Response.Status) PRESTO_PAGES(io.prestosql.PrestoMediaTypes.PRESTO_PAGES) TaskState(io.prestosql.execution.TaskState) POST(javax.ws.rs.POST) Executor(java.util.concurrent.Executor) DateTime(org.joda.time.DateTime) Page(io.prestosql.spi.Page) Futures(com.google.common.util.concurrent.Futures) APPLICATION_JACKSON_SMILE(io.prestosql.protocol.SmileHeader.APPLICATION_JACKSON_SMILE) TaskStatus.initialTaskStatus(io.prestosql.execution.TaskStatus.initialTaskStatus) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Duration(io.airlift.units.Duration) AsyncResponse(javax.ws.rs.container.AsyncResponse) Response(javax.ws.rs.core.Response) AsyncResponseHandler.bindAsyncResponse(io.airlift.jaxrs.AsyncResponseHandler.bindAsyncResponse) BufferResult(io.prestosql.execution.buffer.BufferResult) SerializedPage(io.hetu.core.transport.execution.buffer.SerializedPage) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Path(javax.ws.rs.Path) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET)

Aggregations

SerializedPage (io.hetu.core.transport.execution.buffer.SerializedPage)32 Page (io.prestosql.spi.Page)17 Test (org.testng.annotations.Test)10 PagesSerde (io.hetu.core.transport.execution.buffer.PagesSerde)9 MarkerPage (io.prestosql.spi.snapshot.MarkerPage)8 ImmutableList (com.google.common.collect.ImmutableList)5 URI (java.net.URI)5 List (java.util.List)4 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)3 BlockEncodingSerdeProvider (io.prestosql.spi.snapshot.BlockEncodingSerdeProvider)3 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)2 DataSize (io.airlift.units.DataSize)2 Duration (io.airlift.units.Duration)2 DataCenterQueryResults (io.prestosql.client.DataCenterQueryResults)2 QueryInfo (io.prestosql.execution.QueryInfo)2 PageBuilder (io.prestosql.spi.PageBuilder)2 BlockBuilder (io.prestosql.spi.block.BlockBuilder)2 Objects.requireNonNull (java.util.Objects.requireNonNull)2 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)2 Matchers.anyObject (org.mockito.Matchers.anyObject)2