Examples with SerializedPage com.facebook.presto.spi.page.SerializedPage used on opensource projects

Example 1 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class PrestoSparkDiskPageInput method loadBroadcastTable.

private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
    try {
        CRC32 checksum = new CRC32();
        ImmutableList.Builder<Page> pages = ImmutableList.builder();
        List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
        shuffle(broadcastTaskFilesInfoCopy);
        for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
            checksum.reset();
            PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
            TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
            log.info("Reading path: " + tempStorageHandle.toString());
            try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
                InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
                Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
                while (pagesIterator.hasNext()) {
                    SerializedPage serializedPage = pagesIterator.next();
                    checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
                    Page deserializedPage = pagesSerde.deserialize(serializedPage);
                    pages.add(deserializedPage);
                    stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
                }
                updateMemory.update();
            }
            if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
                throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
            }
        }
        return pages.build();
    } catch (UncheckedIOException | IOException e) {
        throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
    }
}

Example 2 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class Query method getNextResult.

private synchronized QueryResults getNextResult(long token, UriInfo uriInfo, String scheme, DataSize targetResultSize) {
    // check if the result for the token have already been created
    Optional<QueryResults> cachedResult = getCachedResult(token);
    if (cachedResult.isPresent()) {
        return cachedResult.get();
    }
    verify(nextToken.isPresent(), "Can not generate next result when next token is not present");
    verify(token == nextToken.getAsLong(), "Expected token to equal next token");
    URI queryHtmlUri = uriInfo.getRequestUriBuilder().scheme(scheme).replacePath("ui/query.html").replaceQuery(queryId.toString()).build();
    // Remove as many pages as possible from the exchange until just greater than DESIRED_RESULT_BYTES
    // NOTE: it is critical that query results are created for the pages removed from the exchange
    // client while holding the lock because the query may transition to the finished state when the
    // last page is removed.  If another thread observes this state before the response is cached
    // the pages will be lost.
    Iterable<List<Object>> data = null;
    try {
        ImmutableList.Builder<RowIterable> pages = ImmutableList.builder();
        long bytes = 0;
        long rows = 0;
        long targetResultBytes = targetResultSize.toBytes();
        while (bytes < targetResultBytes) {
            SerializedPage serializedPage = exchangeClient.pollPage();
            if (serializedPage == null) {
                break;
            }
            Page page = serde.deserialize(serializedPage);
            bytes += page.getLogicalSizeInBytes();
            rows += page.getPositionCount();
            pages.add(new RowIterable(session.toConnectorSession(), types, page));
        }
        if (rows > 0) {
            // client implementations do not properly handle empty list of data
            data = Iterables.concat(pages.build());
            hasProducedResult = true;
        }
    } catch (Throwable cause) {
        queryManager.failQuery(queryId, cause);
    }
    // get the query info before returning
    // force update if query manager is closed
    QueryInfo queryInfo = queryManager.getFullQueryInfo(queryId);
    queryManager.recordHeartbeat(queryId);
    // grab the update count for non-queries
    if ((data != null) && (queryInfo.getUpdateType() != null) && (updateCount == null) && (columns.size() == 1) && (columns.get(0).getType().equals(StandardTypes.BIGINT))) {
        Iterator<List<Object>> iterator = data.iterator();
        if (iterator.hasNext()) {
            Number number = (Number) iterator.next().get(0);
            if (number != null) {
                updateCount = number.longValue();
            }
        }
    }
    closeExchangeClientIfNecessary(queryInfo);
    // for queries with no output, return a fake result for clients that require it
    if ((queryInfo.getState() == QueryState.FINISHED) && !queryInfo.getOutputStage().isPresent()) {
        columns = ImmutableList.of(new Column("result", BooleanType.BOOLEAN));
        data = ImmutableSet.of(ImmutableList.of(true));
    }
    // (2)there is more data to send (due to buffering)
    if ((!queryInfo.isFinalQueryInfo() && queryInfo.getState() != FAILED) || !exchangeClient.isClosed()) {
        nextToken = OptionalLong.of(token + 1);
    } else {
        nextToken = OptionalLong.empty();
    }
    URI nextResultsUri = null;
    if (nextToken.isPresent()) {
        nextResultsUri = createNextResultsUri(scheme, uriInfo, nextToken.getAsLong());
    }
    // update catalog, schema, and path
    setCatalog = queryInfo.getSetCatalog();
    setSchema = queryInfo.getSetSchema();
    // update setSessionProperties
    setSessionProperties = queryInfo.getSetSessionProperties();
    resetSessionProperties = queryInfo.getResetSessionProperties();
    // update setRoles
    setRoles = queryInfo.getSetRoles();
    // update preparedStatements
    addedPreparedStatements = queryInfo.getAddedPreparedStatements();
    deallocatedPreparedStatements = queryInfo.getDeallocatedPreparedStatements();
    // update startedTransactionId
    startedTransactionId = queryInfo.getStartedTransactionId();
    clearTransactionId = queryInfo.isClearTransactionId();
    // update sessionFunctions
    addedSessionFunctions = queryInfo.getAddedSessionFunctions();
    removedSessionFunctions = queryInfo.getRemovedSessionFunctions();
    // first time through, self is null
    QueryResults queryResults = new QueryResults(queryId.toString(), queryHtmlUri, findCancelableLeafStage(queryInfo), nextResultsUri, columns, data, toStatementStats(queryInfo), toQueryError(queryInfo), queryInfo.getWarnings(), queryInfo.getUpdateType(), updateCount);
    // cache the new result
    lastToken = token;
    lastResult = queryResults;
    return queryResults;
}

Example 3 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class TaskResource method getResults.

@GET
@Path("{taskId}/results/{bufferId}/{token}")
@Produces(PRESTO_PAGES)
public void getResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @PathParam("token") final long token, @HeaderParam(PRESTO_MAX_SIZE) DataSize maxSize, @Suspended AsyncResponse asyncResponse) {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    long start = System.nanoTime();
    ListenableFuture<BufferResult> bufferResultFuture = taskManager.getTaskResults(taskId, bufferId, token, maxSize);
    Duration waitTime = randomizeWaitTime(DEFAULT_MAX_WAIT_TIME);
    bufferResultFuture = addTimeout(bufferResultFuture, () -> BufferResult.emptyResults(taskManager.getTaskInstanceId(taskId), token, false), waitTime, timeoutExecutor);
    ListenableFuture<Response> responseFuture = Futures.transform(bufferResultFuture, result -> {
        List<SerializedPage> serializedPages = result.getSerializedPages();
        GenericEntity<?> entity = null;
        Status status;
        if (serializedPages.isEmpty()) {
            status = Status.NO_CONTENT;
        } else {
            entity = new GenericEntity<>(serializedPages, new TypeToken<List<Page>>() {
            }.getType());
            status = Status.OK;
        }
        return Response.status(status).entity(entity).header(PRESTO_TASK_INSTANCE_ID, result.getTaskInstanceId()).header(PRESTO_PAGE_TOKEN, result.getToken()).header(PRESTO_PAGE_NEXT_TOKEN, result.getNextToken()).header(PRESTO_BUFFER_COMPLETE, result.isBufferComplete()).build();
    }, directExecutor());
    // For hard timeout, add an additional time to max wait for thread scheduling contention and GC
    Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS);
    bindAsyncResponse(asyncResponse, responseFuture, responseExecutor).withTimeout(timeout, Response.status(Status.NO_CONTENT).header(PRESTO_TASK_INSTANCE_ID, taskManager.getTaskInstanceId(taskId)).header(PRESTO_PAGE_TOKEN, token).header(PRESTO_PAGE_NEXT_TOKEN, token).header(PRESTO_BUFFER_COMPLETE, false).build());
    responseFuture.addListener(() -> readFromOutputBufferTime.add(Duration.nanosSince(start)), directExecutor());
    asyncResponse.register((CompletionCallback) throwable -> resultsRequestTime.add(Duration.nanosSince(start)));
}

Example 4 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class TestTempStorageSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    File spillPath = new File(tempDirectory, UUID.randomUUID().toString());
    TempStorageSingleStreamSpillerFactory spillerFactory = new TempStorageSingleStreamSpillerFactory(new TestingTempStorageManager(spillPath.toString()), // executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new BlockEncodingManager(), new SpillerStats(), compression, encryption, LocalTempStorage.NAME);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, new TestingSpillContext(), memoryContext);
    assertTrue(singleStreamSpiller instanceof TempStorageSingleStreamSpiller);
    TempStorageSingleStreamSpiller spiller = (TempStorageSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    int retainedSizeForEmptyDataSink = toIntExact(new OutputStreamDataSink(new DynamicSliceOutput(0)).getRetainedSizeInBytes());
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<SerializedPage> serializedPages = PagesSerdeUtil.readSerializedPages(new InputStreamSliceInput(is));
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        byte markers = serializedPages.next().getPageCodecMarkers();
        assertEquals(PageCodecMarker.COMPRESSED.isSet(markers), compression);
        assertEquals(PageCodecMarker.ENCRYPTED.isSet(markers), encryption);
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 5 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class PageBufferClient method sendGetResults.

private synchronized void sendGetResults(DataSize maxResponseSize) {
    URI uriBase = asyncPageTransportLocation.orElse(location);
    URI uri = HttpUriBuilder.uriBuilderFrom(uriBase).appendPath(String.valueOf(token)).build();
    ListenableFuture<PagesResponse> resultFuture = resultClient.getResults(token, maxResponseSize);
    future = resultFuture;
    Futures.addCallback(resultFuture, new FutureCallback<PagesResponse>() {

        @Override
        public void onSuccess(PagesResponse result) {
            checkNotHoldsLock(this);
            backoff.success();
            List<SerializedPage> pages;
            boolean pagesAccepted;
            try {
                boolean shouldAcknowledge = false;
                synchronized (PageBufferClient.this) {
                    if (taskInstanceId == null) {
                        taskInstanceId = result.getTaskInstanceId();
                    }
                    if (!isNullOrEmpty(taskInstanceId) && !result.getTaskInstanceId().equals(taskInstanceId)) {
                        // TODO: update error message
                        throw new PrestoException(REMOTE_TASK_MISMATCH, format("%s (%s)", REMOTE_TASK_MISMATCH_ERROR, fromUri(uri)));
                    }
                    if (result.getToken() == token) {
                        pages = result.getPages();
                        token = result.getNextToken();
                        shouldAcknowledge = pages.size() > 0;
                    } else {
                        pages = ImmutableList.of();
                    }
                }
                if (shouldAcknowledge && acknowledgePages) {
                    // Acknowledge token without handling the response.
                    // The next request will also make sure the token is acknowledged.
                    // This is to fast release the pages on the buffer side.
                    resultClient.acknowledgeResultsAsync(result.getNextToken());
                }
                for (SerializedPage page : pages) {
                    if (!isChecksumValid(page)) {
                        throw new PrestoException(SERIALIZED_PAGE_CHECKSUM_ERROR, format("Received corrupted serialized page from host %s", HostAddress.fromUri(uri)));
                    }
                }
                // add pages:
                // addPages must be called regardless of whether pages is an empty list because
                // clientCallback can keep stats of requests and responses. For example, it may
                // keep track of how often a client returns empty response and adjust request
                // frequency or buffer size.
                pagesAccepted = clientCallback.addPages(PageBufferClient.this, pages);
            } catch (PrestoException e) {
                handleFailure(e, resultFuture);
                return;
            }
            // update client stats
            if (!pages.isEmpty()) {
                int pageCount = pages.size();
                long rowCount = pages.stream().mapToLong(SerializedPage::getPositionCount).sum();
                if (pagesAccepted) {
                    pagesReceived.addAndGet(pageCount);
                    rowsReceived.addAndGet(rowCount);
                } else {
                    pagesRejected.addAndGet(pageCount);
                    rowsRejected.addAndGet(rowCount);
                }
            }
            requestsCompleted.incrementAndGet();
            synchronized (PageBufferClient.this) {
                // client is complete, acknowledge it by sending it a delete in the next request
                if (result.isClientComplete()) {
                    completed = true;
                }
                if (future == resultFuture) {
                    future = null;
                }
                lastUpdate = DateTime.now();
            }
            clientCallback.requestComplete(PageBufferClient.this);
        }

        @Override
        public void onFailure(Throwable t) {
            log.debug("Request to %s failed %s", uri, t);
            checkNotHoldsLock(this);
            t = resultClient.rewriteException(t);
            if (!(t instanceof PrestoException) && backoff.failure()) {
                String message = format("%s (%s - %s failures, failure duration %s, total failed request time %s)", WORKER_NODE_ERROR, uri, backoff.getFailureCount(), backoff.getFailureDuration().convertTo(SECONDS), backoff.getFailureRequestTimeTotal().convertTo(SECONDS));
                t = new PageTransportTimeoutException(fromUri(uri), message, t);
            }
            handleFailure(t, resultFuture);
        }
    }, pageBufferClientCallbackExecutor);
}