Examples with CompactablePageIO org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO used on opensource projects

Example 1 with CompactablePageIO

use of org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO in project ignite by apache.

the class CompressionProcessorImpl method doCompactPage.

/**
 * @param page Page buffer.
 * @param pageSize Page size.
 * @return Compacted page buffer.
 */
private ByteBuffer doCompactPage(ByteBuffer page, int pageSize) throws IgniteCheckedException {
    PageIO io = PageIO.getPageIO(page);
    ByteBuffer compactPage = compactBuf.get();
    if (io instanceof CompactablePageIO) {
        // Drop the garbage from the page.
        ((CompactablePageIO) io).compactPage(page, compactPage, pageSize);
    } else {
        // Direct buffer is required as output of this method.
        if (page.isDirect())
            return page;
        PageUtils.putBytes(GridUnsafe.bufferAddress(compactPage), 0, page.array());
        compactPage.limit(pageSize);
    }
    return compactPage;
}

Example 2 with CompactablePageIO

use of org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO in project ignite by apache.

the class CompressionProcessorImpl method decompressPage.

/**
 * {@inheritDoc}
 */
@Override
public void decompressPage(ByteBuffer page, int pageSize) throws IgniteCheckedException {
    assert page.capacity() >= pageSize : "capacity=" + page.capacity() + ", pageSize=" + pageSize;
    byte compressType = PageIO.getCompressionType(page);
    if (compressType == UNCOMPRESSED_PAGE)
        // Nothing to do.
        return;
    short compressedSize = PageIO.getCompressedSize(page);
    short compactSize = PageIO.getCompactedSize(page);
    assert compactSize <= pageSize && compactSize >= compressedSize;
    if (compressType == COMPACTED_PAGE) {
        // Just setup bounds before restoring the page.
        page.position(0).limit(compactSize);
    } else {
        ByteBuffer dst = compressBuf.get();
        // Position on a part that needs to be decompressed.
        page.limit(compressedSize).position(PageIO.COMMON_HEADER_END);
        // LZ4 needs this limit to be exact.
        dst.limit(compactSize - PageIO.COMMON_HEADER_END);
        switch(compressType) {
            case ZSTD_COMPRESSED_PAGE:
                Zstd.decompress(dst, page);
                dst.flip();
                break;
            case LZ4_COMPRESSED_PAGE:
                Lz4.decompress(page, dst);
                dst.flip();
                break;
            case SNAPPY_COMPRESSED_PAGE:
                try {
                    Snappy.uncompress(page, dst);
                } catch (IOException e) {
                    throw new IgniteException(e);
                }
                break;
            default:
                throw new IgniteException("Unknown compression: " + compressType);
        }
        page.position(PageIO.COMMON_HEADER_END).limit(compactSize);
        page.put(dst).flip();
        assert page.limit() == compactSize;
    }
    PageIO io = PageIO.getPageIO(page);
    if (io instanceof CompactablePageIO)
        ((CompactablePageIO) io).restorePage(page, pageSize);
    else {
        assert compactSize == pageSize : "Wrong compacted page size [compactSize=" + compactSize + ", pageSize=" + pageSize + ']';
    }
    setCompressionInfo(page, DiskPageCompression.DISABLED, 0, 0);
}

Example 3 with CompactablePageIO

use of org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO in project ignite by apache.

the class PageMemoryTracker method comparePages.

/**
 * Compare pages content.
 *
 * @param fullPageId Full page ID.
 * @param expPage Expected page.
 * @param actualPageAddr Actual page address.
 * @return {@code True} if pages are equals, {@code False} otherwise.
 * @throws IgniteCheckedException If fails.
 */
private boolean comparePages(FullPageId fullPageId, DirectMemoryPage expPage, long actualPageAddr) throws IgniteCheckedException {
    long expPageAddr = expPage.address();
    GridCacheProcessor cacheProc = gridCtx.cache();
    ByteBuffer locBuf = GridUnsafe.wrapPointer(expPageAddr, pageSize);
    ByteBuffer rmtBuf = GridUnsafe.wrapPointer(actualPageAddr, pageSize);
    PageIO pageIo = PageIO.getPageIO(actualPageAddr);
    if (pageIo.getType() == T_DATA_REF_MVCC_LEAF || pageIo.getType() == T_CACHE_ID_DATA_REF_MVCC_LEAF) {
        assert cacheProc.cacheGroup(fullPageId.groupId()).mvccEnabled();
        AbstractDataLeafIO io = (AbstractDataLeafIO) pageIo;
        int cnt = io.getMaxCount(actualPageAddr, pageSize);
        // Reset lock info as there is no sense to log it into WAL.
        for (int i = 0; i < cnt; i++) {
            io.setMvccLockCoordinatorVersion(expPageAddr, i, io.getMvccLockCoordinatorVersion(actualPageAddr, i));
            io.setMvccLockCounter(expPageAddr, i, io.getMvccLockCounter(actualPageAddr, i));
        }
    }
    // Compare only meaningful data.
    if (pageIo instanceof CompactablePageIO) {
        tmpBuf1.clear();
        tmpBuf2.clear();
        ((CompactablePageIO) pageIo).compactPage(locBuf, tmpBuf1, pageSize);
        ((CompactablePageIO) pageIo).compactPage(rmtBuf, tmpBuf2, pageSize);
        locBuf = tmpBuf1;
        rmtBuf = tmpBuf2;
    }
    if (!locBuf.equals(rmtBuf)) {
        log.error("Page buffers are not equals [fullPageId=" + fullPageId + ", pageIo=" + pageIo + ']');
        dumpDiff(locBuf, rmtBuf);
        dumpHistory(expPage);
        return false;
    }
    return true;
}

Example 4 with CompactablePageIO

use of org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO in project ignite by apache.

the class IgniteWalRecoveryTest method testApplyDeltaRecords.

/**
 * @throws Exception if failed.
 */
@Test
public void testApplyDeltaRecords() throws Exception {
    IgniteEx ignite0 = (IgniteEx) startGrid("node0");
    ignite0.cluster().active(true);
    IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
    for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
    GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    db.waitForCheckpoint("test");
    db.enableCheckpoints(false).get();
    // Log something to know where to start.
    WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
    info("Replay marker: " + ptr);
    for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
    info("Done removes...");
    for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    Map<FullPageId, byte[]> rolledPages = new HashMap<>();
    int pageSize = sharedCtx.database().pageSize();
    ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
    buf.order(ByteOrder.nativeOrder());
    // Now check that deltas can be correctly applied.
    try (WALIterator it = sharedCtx.wal().replay(ptr)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof PageSnapshot) {
                PageSnapshot page = (PageSnapshot) rec;
                CacheGroupContext gctx = sharedCtx.cache().cacheGroup(page.groupId());
                int realPageSize = gctx == null ? pageSize : gctx.dataRegion().pageMemory().realPageSize(page.groupId());
                byte[] pageData = page.pageData();
                if (pageData.length < realPageSize) {
                    buf.clear();
                    buf.put(pageData);
                    buf.flip();
                    sharedCtx.kernalContext().compress().decompressPage(buf, realPageSize);
                    pageData = new byte[realPageSize];
                    buf.position(0);
                    buf.get(pageData);
                }
                rolledPages.put(page.fullPageId(), pageData);
            } else if (rec instanceof PageDeltaRecord) {
                PageDeltaRecord delta = (PageDeltaRecord) rec;
                FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
                byte[] pageData = rolledPages.get(fullId);
                if (pageData == null) {
                    pageData = new byte[pageSize];
                    rolledPages.put(fullId, pageData);
                }
                assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
                buf.clear();
                buf.put(pageData);
                buf.flip();
                delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
                buf.get(pageData);
            }
        }
    }
    info("Done apply...");
    PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
    ByteBuffer bufWal = ByteBuffer.allocateDirect(pageSize);
    for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
        FullPageId fullId = entry.getKey();
        ignite0.context().cache().context().database().checkpointReadLock();
        try {
            long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), IoStatisticsHolderNoOp.INSTANCE, true);
            try {
                long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
                try {
                    byte[] data = entry.getValue();
                    if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
                        // Skip tracking pages.
                        continue;
                    // compacted pages in case of compaction is used.
                    if (walPageCompression != null && PageIO.getPageIO(bufPtr) instanceof CompactablePageIO) {
                        CompactablePageIO pageIO = PageIO.getPageIO(bufPtr);
                        buf.clear();
                        bufWal.clear();
                        int realPageSize = data.length;
                        pageIO.compactPage(GridUnsafe.wrapPointer(bufPtr, realPageSize), buf, realPageSize);
                        pageIO.compactPage(ByteBuffer.wrap(data).order(ByteOrder.nativeOrder()), bufWal, realPageSize);
                        bufPtr = GridUnsafe.bufferAddress(buf);
                        data = new byte[bufWal.limit()];
                        bufWal.rewind();
                        bufWal.get(data);
                    }
                    for (int i = 0; i < data.length; i++) assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
                } finally {
                    pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
                }
            } finally {
                pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        } finally {
            ignite0.context().cache().context().database().checkpointReadUnlock();
        }
    }
    ignite0.close();
}