Examples with FullPageId org.apache.ignite.internal.pagememory.FullPageId used on opensource projects

Example 1 with FullPageId

use of org.apache.ignite.internal.pagememory.FullPageId in project ignite-3 by apache.

the class PageMemoryNoLoadSelfTest method testPageTearingSequential.

@Test
public void testPageTearingSequential() throws Exception {
    PageMemory mem = memory();
    mem.start();
    try {
        int pagesCnt = 1024;
        List<FullPageId> pages = new ArrayList<>(pagesCnt);
        for (int i = 0; i < pagesCnt; i++) {
            FullPageId fullId = allocatePage(mem);
            pages.add(fullId);
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                if (i % 64 == 0) {
                    log.info("Writing page [idx=" + i + ", pageId=" + fullId.pageId() + ", page=" + page + ']');
                }
                writePage(mem, fullId, page, i + 1);
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
        for (int i = 0; i < pagesCnt; i++) {
            FullPageId fullId = pages.get(i);
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                if (i % 64 == 0) {
                    log.info("Reading page [idx=" + i + ", pageId=" + fullId.pageId() + ", page=" + page + ']');
                }
                readPage(mem, fullId.pageId(), page, i + 1);
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
    } finally {
        mem.stop(true);
    }
}

Example 2 with FullPageId

use of org.apache.ignite.internal.pagememory.FullPageId in project ignite-3 by apache.

the class PageMemoryNoLoadSelfTest method testPageTearingInner.

@Test
public void testPageTearingInner() throws Exception {
    PageMemory mem = memory();
    mem.start();
    try {
        FullPageId fullId1 = allocatePage(mem);
        FullPageId fullId2 = allocatePage(mem);
        long page1 = mem.acquirePage(fullId1.groupId(), fullId1.pageId());
        try {
            long page2 = mem.acquirePage(fullId2.groupId(), fullId2.pageId());
            log.info("Allocated pages [page1Id=" + fullId1.pageId() + ", page1=" + page1 + ", page2Id=" + fullId2.pageId() + ", page2=" + page2 + ']');
            try {
                writePage(mem, fullId1, page1, 1);
                writePage(mem, fullId2, page2, 2);
                readPage(mem, fullId1.pageId(), page1, 1);
                readPage(mem, fullId2.pageId(), page2, 2);
                // Check read after read.
                readPage(mem, fullId1.pageId(), page1, 1);
                readPage(mem, fullId2.pageId(), page2, 2);
            } finally {
                mem.releasePage(fullId2.groupId(), fullId2.pageId(), page2);
            }
        } finally {
            mem.releasePage(fullId1.groupId(), fullId1.pageId(), page1);
        }
    } finally {
        mem.stop(true);
    }
}

Example 3 with FullPageId

use of org.apache.ignite.internal.pagememory.FullPageId in project ignite-3 by apache.

the class PageMemoryImpl method acquirePage.

private long acquirePage(int grpId, long pageId, IoStatisticsHolder statHolder, boolean restore, @Nullable AtomicBoolean pageAllocated) throws IgniteInternalCheckedException {
    assert started;
    int partId = partitionId(pageId);
    Segment seg = segment(grpId, pageId);
    seg.readLock().lock();
    try {
        long relPtr = seg.loadedPages.get(grpId, effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, INVALID_REL_PTR);
        // The page is loaded to the memory.
        if (relPtr != INVALID_REL_PTR) {
            long absPtr = seg.absolute(relPtr);
            seg.acquirePage(absPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
            return absPtr;
        }
    } finally {
        seg.readLock().unlock();
    }
    FullPageId fullId = new FullPageId(pageId, grpId);
    seg.writeLock().lock();
    long lockedPageAbsPtr = -1;
    boolean readPageFromStore = false;
    try {
        // Double-check.
        long relPtr = seg.loadedPages.get(grpId, fullId.effectivePageId(), seg.partGeneration(grpId, partId), INVALID_REL_PTR, OUTDATED_REL_PTR);
        long absPtr;
        if (relPtr == INVALID_REL_PTR) {
            relPtr = seg.borrowOrAllocateFreePage(pageId);
            if (pageAllocated != null) {
                pageAllocated.set(true);
            }
            if (relPtr == INVALID_REL_PTR) {
                relPtr = seg.removePageForReplacement();
            }
            absPtr = seg.absolute(relPtr);
            fullPageId(absPtr, fullId);
            writeTimestamp(absPtr, coarseCurrentTimeMillis());
            assert !isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + hexLong(relPtr) + ", absPtr=" + hexLong(absPtr) + ']';
            // We can clear dirty flag after the page has been allocated.
            setDirty(fullId, absPtr, false, false);
            seg.pageReplacementPolicy.onMiss(relPtr);
            seg.loadedPages.put(grpId, fullId.effectivePageId(), relPtr, seg.partGeneration(grpId, partId));
            long pageAddr = absPtr + PAGE_OVERHEAD;
            if (!restore) {
                readPageFromStore = true;
            } else {
                setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
                // Must init page ID in order to ensure RWLock tag consistency.
                setPageId(pageAddr, pageId);
            }
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, tag(pageId));
            if (readPageFromStore) {
                boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, TAG_LOCK_ALWAYS);
                assert locked : "Page ID " + fullId + " expected to be locked";
                lockedPageAbsPtr = absPtr;
            }
        } else if (relPtr == OUTDATED_REL_PTR) {
            assert pageIndex(pageId) == 0 : fullId;
            relPtr = seg.refreshOutdatedPage(grpId, pageId, false);
            absPtr = seg.absolute(relPtr);
            long pageAddr = absPtr + PAGE_OVERHEAD;
            setMemory(pageAddr, pageSize(), (byte) 0);
            fullPageId(absPtr, fullId);
            writeTimestamp(absPtr, coarseCurrentTimeMillis());
            setPageId(pageAddr, pageId);
            assert !isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + hexLong(relPtr) + ", absPtr=" + hexLong(absPtr) + ']';
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, tag(pageId));
            seg.pageReplacementPolicy.onRemove(relPtr);
            seg.pageReplacementPolicy.onMiss(relPtr);
        } else {
            absPtr = seg.absolute(relPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
        }
        seg.acquirePage(absPtr);
        if (!readPageFromStore) {
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
        }
        return absPtr;
    } finally {
        seg.writeLock().unlock();
        if (readPageFromStore) {
            assert lockedPageAbsPtr != -1 : "Page is expected to have a valid address [pageId=" + fullId + ", lockedPageAbsPtr=" + hexLong(lockedPageAbsPtr) + ']';
            assert isPageWriteLocked(lockedPageAbsPtr) : "Page is expected to be locked: [pageId=" + fullId + "]";
            long pageAddr = lockedPageAbsPtr + PAGE_OVERHEAD;
            ByteBuffer buf = wrapPointer(pageAddr, pageSize());
            long actualPageId = 0;
            try {
                pmPageMgr.read(grpId, pageId, buf, false);
                statHolder.trackPhysicalAndLogicalRead(pageAddr);
                actualPageId = getPageId(buf);
            } finally {
                rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, actualPageId == 0 ? TAG_LOCK_ALWAYS : tag(actualPageId));
            }
        }
    }
}

Example 4 with FullPageId

use of org.apache.ignite.internal.pagememory.FullPageId in project ignite-3 by apache.

the class RandomLruPageReplacementPolicy method tryToFindSequentially.

/**
 * Will scan all segment pages to find one to evict it.
 *
 * @param cap Capacity.
 */
private long tryToFindSequentially(int cap) throws IgniteInternalCheckedException {
    assert seg.getWriteHoldCount() > 0;
    long prevAddr = INVALID_REL_PTR;
    LoadedPagesMap loadedPages = seg.loadedPages();
    for (int i = 0; i < cap; i++) {
        final ReplaceCandidate nearest = loadedPages.getNearestAt(i);
        assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
        final long addr = nearest.relativePointer();
        int partGen = nearest.generation();
        final long absPageAddr = seg.absolute(addr);
        FullPageId fullId = PageHeader.fullPageId(absPageAddr);
        if (partGen < seg.partGeneration(fullId.groupId(), partitionId(fullId.pageId()))) {
            return seg.refreshOutdatedPage(fullId.groupId(), fullId.pageId(), true);
        }
        boolean pinned = PageHeader.isAcquired(absPageAddr);
        if (addr == prevAddr || pinned) {
            continue;
        }
        final long absEvictAddr = seg.absolute(addr);
        final FullPageId fullPageId = PageHeader.fullPageId(absEvictAddr);
        if (seg.tryToRemovePage(fullPageId, absEvictAddr)) {
            return addr;
        }
        prevAddr = addr;
    }
    throw seg.oomException("no pages to replace");
}

Example 5 with FullPageId

use of org.apache.ignite.internal.pagememory.FullPageId in project ignite-3 by apache.

the class RandomLruPageReplacementPolicy method replace.

/**
 * {@inheritDoc}
 */
@Override
public long replace() throws IgniteInternalCheckedException {
    final ThreadLocalRandom rnd = ThreadLocalRandom.current();
    LoadedPagesMap loadedPages = seg.loadedPages();
    PagePool pool = seg.pool();
    final int cap = loadedPages.capacity();
    // With big number of random picked pages we may fall into infinite loop, because
    // every time the same page may be found.
    Set<Long> ignored = null;
    long relRmvAddr = INVALID_REL_PTR;
    int iterations = 0;
    while (true) {
        long cleanAddr = INVALID_REL_PTR;
        long cleanTs = Long.MAX_VALUE;
        long dirtyAddr = INVALID_REL_PTR;
        long dirtyTs = Long.MAX_VALUE;
        long metaAddr = INVALID_REL_PTR;
        long metaTs = Long.MAX_VALUE;
        for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
            ++iterations;
            if (iterations > pool.pages() * FULL_SCAN_THRESHOLD) {
                break;
            }
            // We need to lookup for pages only in current segment for thread safety,
            // so peeking random memory will lead to checking for found page segment.
            // It's much faster to check available pages for segment right away.
            ReplaceCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap));
            assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
            long rndAddr = nearest.relativePointer();
            int partGen = nearest.generation();
            final long absPageAddr = seg.absolute(rndAddr);
            FullPageId fullId = PageHeader.fullPageId(absPageAddr);
            // Check page mapping consistency.
            assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() + ", actual=" + fullId + ", nearest=" + nearest;
            boolean outdated = partGen < seg.partGeneration(fullId.groupId(), partitionId(fullId.pageId()));
            if (outdated) {
                return seg.refreshOutdatedPage(fullId.groupId(), fullId.pageId(), true);
            }
            boolean pinned = PageHeader.isAcquired(absPageAddr);
            boolean skip = ignored != null && ignored.contains(rndAddr);
            final boolean dirty = PageHeader.dirty(absPageAddr);
            if (relRmvAddr == rndAddr || pinned || skip || fullId.pageId() == META_PAGE_ID || dirty) {
                i--;
                continue;
            }
            final long pageTs = PageHeader.readTimestamp(absPageAddr);
            final boolean storMeta = isStoreMetadataPage(absPageAddr);
            if (pageTs < cleanTs && !dirty && !storMeta) {
                cleanAddr = rndAddr;
                cleanTs = pageTs;
            } else if (pageTs < dirtyTs && dirty && !storMeta) {
                dirtyAddr = rndAddr;
                dirtyTs = pageTs;
            } else if (pageTs < metaTs && storMeta) {
                metaAddr = rndAddr;
                metaTs = pageTs;
            }
            if (cleanAddr != INVALID_REL_PTR) {
                relRmvAddr = cleanAddr;
            } else if (dirtyAddr != INVALID_REL_PTR) {
                relRmvAddr = dirtyAddr;
            } else {
                relRmvAddr = metaAddr;
            }
        }
        if (relRmvAddr == INVALID_REL_PTR) {
            return tryToFindSequentially(cap);
        }
        final long absRmvAddr = seg.absolute(relRmvAddr);
        final FullPageId fullPageId = PageHeader.fullPageId(absRmvAddr);
        if (!seg.tryToRemovePage(fullPageId, absRmvAddr)) {
            if (iterations > 10) {
                if (ignored == null) {
                    ignored = new HashSet<>();
                }
                ignored.add(relRmvAddr);
            }
            if (iterations > seg.pool().pages() * FULL_SCAN_THRESHOLD) {
                return tryToFindSequentially(cap);
            }
            continue;
        }
        return relRmvAddr;
    }
}