Examples with Page io.trino.spi.Page used on opensource projects

Example 16 with Page

use of io.trino.spi.Page in project trino by trinodb.

the class PageSourceOperator method getOutput.

@Override
public Page getOutput() {
    Page page = pageSource.getNextPage();
    if (page == null) {
        return null;
    }
    // update operator stats
    long endCompletedBytes = pageSource.getCompletedBytes();
    long endReadTimeNanos = pageSource.getReadTimeNanos();
    operatorContext.recordPhysicalInputWithTiming(endCompletedBytes - completedBytes, page.getPositionCount(), endReadTimeNanos - readTimeNanos);
    operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount());
    completedBytes = endCompletedBytes;
    readTimeNanos = endReadTimeNanos;
    // assure the page is in memory before handing to another operator
    page = page.getLoadedPage();
    return page;
}

Example 17 with Page

use of io.trino.spi.Page in project trino by trinodb.

the class IndexSnapshotBuilder method createIndexSnapshot.

public IndexSnapshot createIndexSnapshot(UnloadedIndexKeyRecordSet indexKeysRecordSet) {
    checkArgument(indexKeysRecordSet.getColumnTypes().equals(missingKeysTypes), "indexKeysRecordSet must have same schema as missingKeys");
    checkState(!isMemoryExceeded(), "Max memory exceeded");
    for (Page page : pages) {
        outputPagesIndex.addPage(page);
    }
    pages.clear();
    LookupSource lookupSource = outputPagesIndex.createLookupSourceSupplier(session, keyOutputChannels, keyOutputHashChannel, Optional.empty(), Optional.empty(), ImmutableList.of()).get();
    // Build a page containing the keys that produced no output rows, so in future requests can skip these keys
    verify(missingKeysPageBuilder.isEmpty());
    UnloadedIndexKeyRecordCursor indexKeysRecordCursor = indexKeysRecordSet.cursor();
    while (indexKeysRecordCursor.advanceNextPosition()) {
        Page page = indexKeysRecordCursor.getPage();
        int position = indexKeysRecordCursor.getPosition();
        if (lookupSource.getJoinPosition(position, page, page) < 0) {
            missingKeysPageBuilder.declarePosition();
            for (int i = 0; i < page.getChannelCount(); i++) {
                Block block = page.getBlock(i);
                Type type = indexKeysRecordCursor.getType(i);
                type.appendTo(block, position, missingKeysPageBuilder.getBlockBuilder(i));
            }
        }
    }
    Page missingKeysPage = missingKeysPageBuilder.build();
    if (!missingKeysPageBuilder.isEmpty()) {
        missingKeysPageBuilder.reset();
    }
    memoryInBytes += missingKeysPage.getSizeInBytes();
    if (isMemoryExceeded()) {
        return null;
    }
    // only update missing keys if we have new missing keys
    if (missingKeysPage.getPositionCount() != 0) {
        missingKeysIndex.addPage(missingKeysPage);
        missingKeys = missingKeysIndex.createLookupSourceSupplier(session, missingKeysChannels).get();
    }
    return new IndexSnapshot(lookupSource, missingKeys);
}

Example 18 with Page

use of io.trino.spi.Page in project trino by trinodb.

the class DefaultPageJoiner method spillJoinProbe.

private void spillJoinProbe(SpillInfoSnapshot spillInfoSnapshot) {
    verifyNotNull(probe, "probe is null");
    verify(pageBuilder.isEmpty(), "pageBuilder must be flushed before spill");
    checkArgument(spillInfoSnapshot.getSpillEpoch() > NO_SPILL_EPOCH, "invalid spill epoch");
    /*
         * Spill state changed. All probe rows that were not processed yet should be treated as regular input (and be partially spilled).
         * If current row maps to the now-spilled partition, it needs to be saved for later. If it maps to a partition still in memory, it
         * should be added together with not-yet-processed rows. In either case we need to resume processing the row starting at its
         * current position in the lookup source.
         */
    if (probe.getPosition() < 0) {
        // Processing of the page hasn't been started yet.
        probe = joinProbeFactory.createJoinProbe(spillAndMaskSpilledPositions(probe.getPage(), spillInfoSnapshot));
    } else {
        int currentRowPartition = partitionGenerator.get().getPartition(probe.getPage(), probe.getPosition());
        boolean currentRowSpilled = spillInfoSnapshot.getSpillMask().test(currentRowPartition);
        if (currentRowSpilled) {
            spilledRows.merge(currentRowPartition, new SavedRow(probe.getPage(), probe.getPosition(), getJoinPositionWithinPartition(), currentProbePositionProducedRow, joinSourcePositions), (oldValue, newValue) -> {
                throw new IllegalStateException(format("Partition %s is already spilled", currentRowPartition));
            });
            Page remaining = pageTail(probe.getPage(), probe.getPosition() + 1);
            // create probe starting from next position
            probe = joinProbeFactory.createJoinProbe(spillAndMaskSpilledPositions(remaining, spillInfoSnapshot));
            resetProbeRowState();
        } else {
            Page remaining = pageTail(probe.getPage(), probe.getPosition());
            // create probe starting from current position and keep current row join state
            probe = joinProbeFactory.createJoinProbe(spillAndMaskSpilledPositions(remaining, spillInfoSnapshot));
            verify(probe.advanceNextPosition());
        }
    }
    spillEpoch = spillInfoSnapshot.getSpillEpoch();
}

Example 19 with Page

use of io.trino.spi.Page in project trino by trinodb.

the class HashBuilderOperator method finishLookupSourceUnspilling.

private void finishLookupSourceUnspilling() {
    checkState(state == State.INPUT_UNSPILLING);
    if (!unspillInProgress.get().isDone()) {
        // Pages have not been unspilled yet.
        return;
    }
    // Use Queue so that Pages already consumed by Index are not retained by us.
    Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get()));
    unspillInProgress = Optional.empty();
    long sizeOfUnspilledPages = pages.stream().mapToLong(Page::getSizeInBytes).sum();
    long retainedSizeOfUnspilledPages = pages.stream().mapToLong(Page::getRetainedSizeInBytes).sum();
    log.debug("Unspilling for operator %s, unspilled partition %d, sizeOfUnspilledPages %s, retainedSizeOfUnspilledPages %s", operatorContext, partitionIndex, succinctBytes(sizeOfUnspilledPages), succinctBytes(retainedSizeOfUnspilledPages));
    localUserMemoryContext.setBytes(retainedSizeOfUnspilledPages + index.getEstimatedSize().toBytes());
    while (!pages.isEmpty()) {
        Page next = pages.remove();
        index.addPage(next);
        // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size.
        retainedSizeOfUnspilledPages -= next.getRetainedSizeInBytes();
        localUserMemoryContext.setBytes(retainedSizeOfUnspilledPages + index.getEstimatedSize().toBytes());
    }
    LookupSourceSupplier partition = buildLookupSource();
    lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one"));
    localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes());
    spilledLookupSourceHandle.setLookupSource(partition);
    state = State.INPUT_UNSPILLED_AND_BUILT;
}

Example 20 with Page

use of io.trino.spi.Page in project trino by trinodb.

the class LookupOuterOperator method getOutput.

@Override
public Page getOutput() {
    if (outerPositions == null) {
        outerPositions = tryGetFutureValue(outerPositionsFuture).orElse(null);
        if (outerPositions == null) {
            return null;
        }
    }
    boolean outputPositionsFinished = false;
    while (!pageBuilder.isFull()) {
        // write build columns
        outputPositionsFinished = !outerPositions.appendToNext(pageBuilder, probeOutputTypes.size());
        if (outputPositionsFinished) {
            break;
        }
        pageBuilder.declarePosition();
        // todo use RLE blocks
        for (int probeChannel = 0; probeChannel < probeOutputTypes.size(); probeChannel++) {
            pageBuilder.getBlockBuilder(probeChannel).appendNull();
        }
    }
    // only flush full pages unless we are done
    Page page = null;
    if (pageBuilder.isFull() || (outputPositionsFinished && !pageBuilder.isEmpty())) {
        page = pageBuilder.build();
        pageBuilder.reset();
    }
    if (outputPositionsFinished) {
        close();
    }
    return page;
}