Examples with SerializedPage com.facebook.presto.spi.page.SerializedPage used on opensource projects

Example 6 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class ExchangeClient method addPages.

private boolean addPages(List<SerializedPage> pages) {
    // Compute stats before acquiring the lock
    long pagesRetainedSizeInBytes = 0;
    long responseSize = 0;
    for (SerializedPage page : pages) {
        pagesRetainedSizeInBytes += page.getRetainedSizeInBytes();
        responseSize += page.getSizeInBytes();
    }
    List<SettableFuture<?>> notify = ImmutableList.of();
    synchronized (this) {
        if (isClosed() || isFailed()) {
            return false;
        }
        if (!pages.isEmpty()) {
            pageBuffer.addAll(pages);
            bufferRetainedSizeInBytes += pagesRetainedSizeInBytes;
            maxBufferRetainedSizeInBytes = max(maxBufferRetainedSizeInBytes, bufferRetainedSizeInBytes);
            systemMemoryContext.setBytes(bufferRetainedSizeInBytes);
            // Notify pending listeners that a page has been added
            notify = ImmutableList.copyOf(blockedCallers);
            blockedCallers.clear();
        }
        successfulRequests++;
        responseSizeExponentialMovingAverage.update(responseSize);
    }
    // Trigger notifications after releasing the lock
    notifyListeners(notify);
    return true;
}

Example 7 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class PartitionedOutputBuffer method enqueue.

@Override
public void enqueue(Lifespan lifespan, int partitionNumber, List<SerializedPage> pages) {
    requireNonNull(lifespan, "lifespan is null");
    requireNonNull(pages, "pages is null");
    checkState(pageTracker.isLifespanCompletionCallbackRegistered(), "lifespanCompletionCallback must be set before enqueueing data");
    // this can happen with a limit query
    if (!state.get().canAddPages() || pageTracker.isNoMorePagesForLifespan(lifespan)) {
        return;
    }
    ImmutableList.Builder<SerializedPageReference> references = ImmutableList.builderWithExpectedSize(pages.size());
    long bytesAdded = 0;
    long rowCount = 0;
    for (SerializedPage page : pages) {
        long retainedSize = page.getRetainedSizeInBytes();
        bytesAdded += retainedSize;
        rowCount += page.getPositionCount();
        // create page reference counts with an initial single reference
        references.add(new SerializedPageReference(page, 1, lifespan));
    }
    List<SerializedPageReference> serializedPageReferences = references.build();
    // reserve memory
    memoryManager.updateMemoryUsage(bytesAdded);
    // update stats
    totalRowsAdded.addAndGet(rowCount);
    totalPagesAdded.addAndGet(serializedPageReferences.size());
    pageTracker.incrementLifespanPageCount(lifespan, serializedPageReferences.size());
    // add pages to the buffer (this will increase the reference count by one)
    partitions.get(partitionNumber).enqueuePages(serializedPageReferences);
    // drop the initial reference
    dereferencePages(serializedPageReferences, pageTracker);
}

Example 8 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class SpoolingOutputBuffer method acknowledgePagesFromMemory.

private synchronized void acknowledgePagesFromMemory(long sequenceId, long startSequenceId) {
    checkState(startSequenceId == currentMemorySequenceId.get(), "Invalid startSequenceId for memory pages");
    int pagesToRemove = toIntExact(sequenceId - startSequenceId);
    checkArgument(pagesToRemove <= pages.size(), "Invalid sequenceId");
    long bytesRemoved = 0;
    for (int i = 0; i < pagesToRemove; i++) {
        SerializedPage removedPage = pages.remove();
        bytesRemoved += removedPage.getRetainedSizeInBytes();
        currentMemorySequenceId.incrementAndGet();
    }
    totalBufferedPages.addAndGet(-pagesToRemove);
    totalBufferedBytes.addAndGet(-bytesRemoved);
    totalInMemoryBytes.addAndGet(-bytesRemoved);
    totalPagesRemaining.addAndGet(-pagesToRemove);
}

Example 9 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class SpoolingOutputBuffer method getPagesFromMemory.

private List<SerializedPage> getPagesFromMemory(long startSequenceId, GetTracker getTracker) {
    checkArgument(startSequenceId == currentMemorySequenceId.get(), "Invalid startSequenceId for memory pages");
    checkArgument(getTracker.bytes < getTracker.maxSize.toBytes(), "bytesRead is greater than maxSize");
    ImmutableList.Builder<SerializedPage> result = ImmutableList.builder();
    List<SerializedPage> pages = getTracker.getMemoryPages();
    long maxBytes = getTracker.maxSize.toBytes();
    long bytes = 0;
    long pageCount = 0;
    for (SerializedPage page : pages) {
        bytes += page.getRetainedSizeInBytes();
        if (pageCount != 0 && bytes > maxBytes) {
            break;
        }
        result.add(page);
        pageCount++;
    }
    return result.build();
}

Example 10 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class ArbitraryOutputBuffer method enqueue.

@Override
public void enqueue(Lifespan lifespan, List<SerializedPage> pages) {
    checkState(!Thread.holdsLock(this), "Can not enqueue pages while holding a lock on this");
    requireNonNull(lifespan, "lifespan is null");
    requireNonNull(pages, "page is null");
    checkState(pageTracker.isLifespanCompletionCallbackRegistered(), "lifespanCompletionCallback must be set before enqueueing data");
    // this can happen with a limit query
    if (!state.get().canAddPages() || pageTracker.isNoMorePagesForLifespan(lifespan)) {
        return;
    }
    ImmutableList.Builder<SerializedPageReference> references = ImmutableList.builderWithExpectedSize(pages.size());
    long bytesAdded = 0;
    long rowCount = 0;
    for (SerializedPage page : pages) {
        long retainedSize = page.getRetainedSizeInBytes();
        bytesAdded += retainedSize;
        rowCount += page.getPositionCount();
        // create page reference counts with an initial single reference
        references.add(new SerializedPageReference(page, 1, lifespan));
    }
    List<SerializedPageReference> serializedPageReferences = references.build();
    // reserve memory
    memoryManager.updateMemoryUsage(bytesAdded);
    // update stats
    totalRowsAdded.addAndGet(rowCount);
    totalPagesAdded.addAndGet(serializedPageReferences.size());
    pageTracker.incrementLifespanPageCount(lifespan, serializedPageReferences.size());
    // add pages to the buffer (this will increase the reference count by one)
    masterBuffer.addPages(serializedPageReferences);
    // process any pending reads from the client buffers
    List<ClientBuffer> buffers = safeGetBuffersSnapshot();
    if (buffers.isEmpty()) {
        return;
    }
    // handle potential for racy update of next index and client buffers present
    int index = nextClientBufferIndex.get() % buffers.size();
    for (int i = 0; i < buffers.size(); i++) {
        if (masterBuffer.isEmpty()) {
            // Resume from the current client buffer on the next iteration
            nextClientBufferIndex.set(index);
            break;
        }
        buffers.get(index).loadPagesIfNecessary(masterBuffer);
        index = (index + 1) % buffers.size();
    }
}