Examples with IgniteDataIntegrityViolationException org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException used on opensource projects

Example 1 with IgniteDataIntegrityViolationException

use of org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException in project ignite by apache.

the class PageMemoryImpl method acquirePage.

/**
 * @param grpId Group id.
 * @param pageId Page id.
 * @param statHolder Stat holder.
 * @param restore Restore.
 * @param pageAllocated Page allocated.
 */
private long acquirePage(int grpId, long pageId, IoStatisticsHolder statHolder, boolean restore, @Nullable AtomicBoolean pageAllocated) throws IgniteCheckedException {
    assert started;
    int partId = PageIdUtils.partId(pageId);
    Segment seg = segment(grpId, pageId);
    seg.readLock().lock();
    try {
        long relPtr = seg.loadedPages.get(grpId, PageIdUtils.effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, INVALID_REL_PTR);
        // The page is loaded to the memory.
        if (relPtr != INVALID_REL_PTR) {
            long absPtr = seg.absolute(relPtr);
            seg.acquirePage(absPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
            return absPtr;
        }
    } finally {
        seg.readLock().unlock();
    }
    FullPageId fullId = new FullPageId(pageId, grpId);
    seg.writeLock().lock();
    long lockedPageAbsPtr = -1;
    boolean readPageFromStore = false;
    try {
        // Double-check.
        long relPtr = seg.loadedPages.get(grpId, fullId.effectivePageId(), seg.partGeneration(grpId, partId), INVALID_REL_PTR, OUTDATED_REL_PTR);
        long absPtr;
        if (relPtr == INVALID_REL_PTR) {
            relPtr = seg.borrowOrAllocateFreePage(pageId);
            if (pageAllocated != null)
                pageAllocated.set(true);
            if (relPtr == INVALID_REL_PTR)
                relPtr = seg.removePageForReplacement();
            absPtr = seg.absolute(relPtr);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            // We can clear dirty flag after the page has been allocated.
            setDirty(fullId, absPtr, false, false);
            seg.pageReplacementPolicy.onMiss(relPtr);
            seg.loadedPages.put(grpId, fullId.effectivePageId(), relPtr, seg.partGeneration(grpId, partId));
            long pageAddr = absPtr + PAGE_OVERHEAD;
            if (!restore) {
                if (delayedPageReplacementTracker != null)
                    delayedPageReplacementTracker.waitUnlock(fullId);
                readPageFromStore = true;
            } else {
                GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
                // Must init page ID in order to ensure RWLock tag consistency.
                PageIO.setPageId(pageAddr, pageId);
            }
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
            if (readPageFromStore) {
                boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
                assert locked : "Page ID " + fullId + " expected to be locked";
                lockedPageAbsPtr = absPtr;
            }
        } else if (relPtr == OUTDATED_REL_PTR) {
            assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
            relPtr = seg.refreshOutdatedPage(grpId, pageId, false);
            absPtr = seg.absolute(relPtr);
            long pageAddr = absPtr + PAGE_OVERHEAD;
            GridUnsafe.setMemory(pageAddr, pageSize(), (byte) 0);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            PageIO.setPageId(pageAddr, pageId);
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
            seg.pageReplacementPolicy.onRemove(relPtr);
            seg.pageReplacementPolicy.onMiss(relPtr);
        } else {
            absPtr = seg.absolute(relPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
        }
        seg.acquirePage(absPtr);
        if (!readPageFromStore)
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
        return absPtr;
    } catch (IgniteOutOfMemoryException oom) {
        ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
        throw oom;
    } finally {
        seg.writeLock().unlock();
        if (delayedPageReplacementTracker != null)
            delayedPageReplacementTracker.delayedPageWrite().finishReplacement();
        if (readPageFromStore) {
            assert lockedPageAbsPtr != -1 : "Page is expected to have a valid address [pageId=" + fullId + ", lockedPageAbsPtr=" + U.hexLong(lockedPageAbsPtr) + ']';
            assert isPageWriteLocked(lockedPageAbsPtr) : "Page is expected to be locked: [pageId=" + fullId + "]";
            long pageAddr = lockedPageAbsPtr + PAGE_OVERHEAD;
            ByteBuffer buf = wrapPointer(pageAddr, pageSize());
            long actualPageId = 0;
            try {
                pmPageMgr.read(grpId, pageId, buf, false);
                statHolder.trackPhysicalAndLogicalRead(pageAddr);
                actualPageId = PageIO.getPageId(buf);
                dataRegionMetrics.onPageRead();
                if (PageIO.isIndexPage(PageIO.getType(buf)))
                    dataRegionMetrics.cacheGrpPageMetrics(grpId).indexPages().increment();
            } catch (IgniteDataIntegrityViolationException e) {
                U.warn(log, "Failed to read page (data integrity violation encountered, will try to " + "restore using existing WAL) [fullPageId=" + fullId + ']', e);
                buf.rewind();
                tryToRestorePage(fullId, buf);
                // Mark the page as dirty because it has been restored.
                setDirty(fullId, lockedPageAbsPtr, true, false);
                // And save the page snapshot in the WAL.
                beforeReleaseWrite(fullId, pageAddr, true);
                statHolder.trackPhysicalAndLogicalRead(pageAddr);
                dataRegionMetrics.onPageRead();
            } finally {
                rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, actualPageId == 0 ? OffheapReadWriteLock.TAG_LOCK_ALWAYS : PageIdUtils.tag(actualPageId));
            }
        }
    }
}

Example 2 with IgniteDataIntegrityViolationException

use of org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException in project ignite by apache.

the class FilePageStore method read.

/**
 * @param pageId Page ID.
 * @param pageBuf Page buffer to read into.
 * @param checkCrc Check CRC on page.
 * @param keepCrc By default reading zeroes CRC which was on file, but you can keep it in pageBuf if set keepCrc
 * @return {@code true} if page has been read successfully, {@code false} if page hasn't been written yet.
 * @throws IgniteCheckedException If reading failed (IO error occurred).
 */
public boolean read(long pageId, ByteBuffer pageBuf, boolean checkCrc, boolean keepCrc) throws IgniteCheckedException {
    init();
    try {
        long off = pageOffset(pageId);
        assert pageBuf.capacity() == pageSize;
        assert pageBuf.remaining() == pageSize;
        assert pageBuf.position() == 0;
        assert pageBuf.order() == ByteOrder.nativeOrder();
        assert off <= allocated.get() : "calculatedOffset=" + off + ", allocated=" + allocated.get() + ", headerSize=" + headerSize() + ", cfgFile=" + pathProvider.apply().toAbsolutePath();
        int n = readWithFailover(pageBuf, off);
        // If page was not written yet, nothing to read.
        if (n < 0) {
            pageBuf.put(new byte[pageBuf.remaining()]);
            return false;
        }
        int savedCrc32 = PageIO.getCrc(pageBuf);
        PageIO.setCrc(pageBuf, 0);
        pageBuf.position(0);
        if (checkCrc) {
            int curCrc32 = FastCrc.calcCrc(pageBuf, getCrcSize(pageId, pageBuf));
            if ((savedCrc32 ^ curCrc32) != 0)
                throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + "[id=" + U.hexLong(pageId) + ", off=" + (off - pageSize) + ", file=" + getFileAbsolutePath() + ", fileSize=" + fileIO.size() + ", savedCrc=" + U.hexInt(savedCrc32) + ", curCrc=" + U.hexInt(curCrc32) + ", page=" + U.toHexString(pageBuf) + "]");
        }
        assert PageIO.getCrc(pageBuf) == 0;
        if (keepCrc)
            PageIO.setCrc(pageBuf, savedCrc32);
        return true;
    } catch (IOException e) {
        throw new StorageException("Failed to read page [file=" + getFileAbsolutePath() + ", pageId=" + pageId + "]", e);
    }
}

Example 3 with IgniteDataIntegrityViolationException

use of org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException in project ignite by apache.

the class PageMemoryImpl method acquirePage.

/**
 * {@inheritDoc}
 */
@Override
public long acquirePage(int grpId, long pageId, boolean restore) throws IgniteCheckedException {
    FullPageId fullId = new FullPageId(pageId, grpId);
    int partId = PageIdUtils.partId(pageId);
    Segment seg = segment(grpId, pageId);
    seg.readLock().lock();
    try {
        long relPtr = seg.loadedPages.get(grpId, PageIdUtils.effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, INVALID_REL_PTR);
        // The page is loaded to the memory.
        if (relPtr != INVALID_REL_PTR) {
            long absPtr = seg.absolute(relPtr);
            seg.acquirePage(absPtr);
            return absPtr;
        }
    } finally {
        seg.readLock().unlock();
    }
    DelayedDirtyPageWrite delayedWriter = delayedPageReplacementTracker != null ? delayedPageReplacementTracker.delayedPageWrite() : null;
    seg.writeLock().lock();
    long lockedPageAbsPtr = -1;
    boolean readPageFromStore = false;
    try {
        // Double-check.
        long relPtr = seg.loadedPages.get(grpId, PageIdUtils.effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, OUTDATED_REL_PTR);
        long absPtr;
        if (relPtr == INVALID_REL_PTR) {
            relPtr = seg.borrowOrAllocateFreePage(pageId);
            if (relPtr == INVALID_REL_PTR)
                relPtr = seg.removePageForReplacement(delayedWriter == null ? flushDirtyPage : delayedWriter);
            absPtr = seg.absolute(relPtr);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            // We can clear dirty flag after the page has been allocated.
            setDirty(fullId, absPtr, false, false);
            seg.loadedPages.put(grpId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(grpId, partId));
            long pageAddr = absPtr + PAGE_OVERHEAD;
            if (!restore) {
                if (delayedPageReplacementTracker != null)
                    delayedPageReplacementTracker.waitUnlock(fullId);
                readPageFromStore = true;
            } else {
                GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
                // Must init page ID in order to ensure RWLock tag consistency.
                PageIO.setPageId(pageAddr, pageId);
            }
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
            if (readPageFromStore) {
                boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
                assert locked : "Page ID " + fullId + " expected to be locked";
                lockedPageAbsPtr = absPtr;
            }
        } else if (relPtr == OUTDATED_REL_PTR) {
            assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
            relPtr = refreshOutdatedPage(seg, grpId, pageId, false);
            absPtr = seg.absolute(relPtr);
            long pageAddr = absPtr + PAGE_OVERHEAD;
            GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            PageIO.setPageId(pageAddr, pageId);
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
        } else
            absPtr = seg.absolute(relPtr);
        seg.acquirePage(absPtr);
        return absPtr;
    } finally {
        seg.writeLock().unlock();
        if (delayedWriter != null)
            delayedWriter.finishReplacement();
        if (readPageFromStore) {
            assert lockedPageAbsPtr != -1 : "Page is expected to have a valid address [pageId=" + fullId + ", lockedPageAbsPtr=" + U.hexLong(lockedPageAbsPtr) + ']';
            assert isPageWriteLocked(lockedPageAbsPtr) : "Page is expected to be locked: [pageId=" + fullId + "]";
            long pageAddr = lockedPageAbsPtr + PAGE_OVERHEAD;
            ByteBuffer buf = wrapPointer(pageAddr, pageSize());
            try {
                storeMgr.read(grpId, pageId, buf);
            } catch (IgniteDataIntegrityViolationException ignore) {
                U.warn(log, "Failed to read page (data integrity violation encountered, will try to " + "restore using existing WAL) [fullPageId=" + fullId + ']');
                buf.rewind();
                tryToRestorePage(fullId, buf);
            } finally {
                rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
            }
        }
    }
}

Example 4 with IgniteDataIntegrityViolationException

use of org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException in project ignite by apache.

the class IndexPagesMetricsPageDisplacementTest method getIdxPagesOnDisk.

/**
 * Returns IDs of index pages currently residing in the storage.
 */
private List<Long> getIdxPagesOnDisk(int grpId) throws IgniteCheckedException {
    FilePageStoreManager pageStoreMgr = (FilePageStoreManager) grid.context().cache().context().pageStore();
    FilePageStore pageStore = (FilePageStore) pageStoreMgr.getStore(grpId, PageIdAllocator.INDEX_PARTITION);
    List<Long> result = new ArrayList<>();
    ByteBuffer buf = ByteBuffer.allocateDirect(pageStore.getPageSize()).order(ByteOrder.nativeOrder());
    // Page Store contains a one-page header
    long numPages = pageStore.size() / pageStore.getPageSize() - 1;
    for (int i = 0; i < numPages; i++) {
        long pageId = PageIdUtils.pageId(PageIdAllocator.INDEX_PARTITION, (byte) 0, i);
        try {
            pageStore.read(pageId, buf, false);
        } catch (IgniteDataIntegrityViolationException ignored) {
        // sometimes we try to access an invalid page, in which case this exception will be thrown.
        // We simply ignore it and try to access other pages.
        }
        if (PageIO.isIndexPage(PageIO.getType(buf)))
            result.add(PageIO.getPageId(buf));
        buf.clear();
    }
    return result;
}