Examples with IgniteOutOfMemoryException org.apache.ignite.internal.mem.IgniteOutOfMemoryException used on opensource projects

Example 1 with IgniteOutOfMemoryException

use of org.apache.ignite.internal.mem.IgniteOutOfMemoryException in project ignite by apache.

the class PageMemoryImpl method allocatePage.

/**
 * {@inheritDoc}
 */
@Override
public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException {
    assert flags != PageIdAllocator.FLAG_IDX && partId <= PageIdAllocator.MAX_PARTITION_ID || flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION : "flags = " + flags + ", partId = " + partId;
    assert started;
    assert stateChecker.checkpointLockIsHeldByThread();
    if (isThrottlingEnabled())
        writeThrottle.onMarkDirty(false);
    long pageId = pmPageMgr.allocatePage(grpId, partId, flags);
    // it's crucial for tracking pages (zero page is super one)
    assert PageIdUtils.pageIndex(pageId) > 0;
    // We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
    // Otherwise it is possible that on file will be empty page which will be saved at snapshot and read with error
    // because there is no crc inside them.
    Segment seg = segment(grpId, pageId);
    seg.writeLock().lock();
    boolean isTrackingPage = changeTracker != null && PageIdUtils.pageIndex(trackingIO.trackingPageFor(pageId, realPageSize(grpId))) == PageIdUtils.pageIndex(pageId);
    if (isTrackingPage && PageIdUtils.flag(pageId) == PageIdAllocator.FLAG_AUX)
        pageId = PageIdUtils.pageId(PageIdUtils.partId(pageId), PageIdAllocator.FLAG_DATA, PageIdUtils.pageIndex(pageId));
    FullPageId fullId = new FullPageId(pageId, grpId);
    try {
        long relPtr = seg.loadedPages.get(grpId, PageIdUtils.effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, OUTDATED_REL_PTR);
        if (relPtr == OUTDATED_REL_PTR) {
            relPtr = seg.refreshOutdatedPage(grpId, pageId, false);
            seg.pageReplacementPolicy.onRemove(relPtr);
        }
        if (relPtr == INVALID_REL_PTR)
            relPtr = seg.borrowOrAllocateFreePage(pageId);
        if (relPtr == INVALID_REL_PTR)
            relPtr = seg.removePageForReplacement();
        long absPtr = seg.absolute(relPtr);
        GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
        PageHeader.fullPageId(absPtr, fullId);
        PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
        rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
        // TODO GG-11480
        assert PageIO.getCrc(absPtr + PAGE_OVERHEAD) == 0;
        assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ", pinCntr=" + PageHeader.pinCount(absPtr) + ']';
        setDirty(fullId, absPtr, true, true);
        if (isTrackingPage) {
            long pageAddr = absPtr + PAGE_OVERHEAD;
            // We can modify page buffer directly.
            if (PageIO.getType(pageAddr) == 0) {
                PageMetrics metrics = dataRegionMetrics.cacheGrpPageMetrics(grpId);
                trackingIO.initNewPage(pageAddr, pageId, realPageSize(grpId), metrics);
                if (!ctx.wal().disabled(fullId.groupId())) {
                    if (!ctx.wal().isAlwaysWriteFullPages())
                        ctx.wal().log(new InitNewPageRecord(grpId, pageId, trackingIO.getType(), trackingIO.getVersion(), pageId));
                    else {
                        ctx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize(), realPageSize(fullId.groupId())));
                    }
                }
            }
        }
        seg.pageReplacementPolicy.onMiss(relPtr);
        seg.loadedPages.put(grpId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(grpId, partId));
    } catch (IgniteOutOfMemoryException oom) {
        DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
        IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out of memory in data region [" + "name=" + dataRegionCfg.getName() + ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) + ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) + ", persistenceEnabled=" + dataRegionCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + "  ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + "  ^-- Enable eviction or expiration policies");
        e.initCause(oom);
        ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
        throw e;
    } finally {
        seg.writeLock().unlock();
    }
    // Finish replacement only when an exception wasn't thrown otherwise it possible to corrupt B+Tree.
    if (delayedPageReplacementTracker != null)
        delayedPageReplacementTracker.delayedPageWrite().finishReplacement();
    // we have allocated 'tracking' page, we need to allocate regular one
    return isTrackingPage ? allocatePage(grpId, partId, flags) : pageId;
}

Example 2 with IgniteOutOfMemoryException

use of org.apache.ignite.internal.mem.IgniteOutOfMemoryException in project ignite by apache.

the class PageMemoryImpl method acquirePage.

/**
 * @param grpId Group id.
 * @param pageId Page id.
 * @param statHolder Stat holder.
 * @param restore Restore.
 * @param pageAllocated Page allocated.
 */
private long acquirePage(int grpId, long pageId, IoStatisticsHolder statHolder, boolean restore, @Nullable AtomicBoolean pageAllocated) throws IgniteCheckedException {
    assert started;
    int partId = PageIdUtils.partId(pageId);
    Segment seg = segment(grpId, pageId);
    seg.readLock().lock();
    try {
        long relPtr = seg.loadedPages.get(grpId, PageIdUtils.effectivePageId(pageId), seg.partGeneration(grpId, partId), INVALID_REL_PTR, INVALID_REL_PTR);
        // The page is loaded to the memory.
        if (relPtr != INVALID_REL_PTR) {
            long absPtr = seg.absolute(relPtr);
            seg.acquirePage(absPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
            return absPtr;
        }
    } finally {
        seg.readLock().unlock();
    }
    FullPageId fullId = new FullPageId(pageId, grpId);
    seg.writeLock().lock();
    long lockedPageAbsPtr = -1;
    boolean readPageFromStore = false;
    try {
        // Double-check.
        long relPtr = seg.loadedPages.get(grpId, fullId.effectivePageId(), seg.partGeneration(grpId, partId), INVALID_REL_PTR, OUTDATED_REL_PTR);
        long absPtr;
        if (relPtr == INVALID_REL_PTR) {
            relPtr = seg.borrowOrAllocateFreePage(pageId);
            if (pageAllocated != null)
                pageAllocated.set(true);
            if (relPtr == INVALID_REL_PTR)
                relPtr = seg.removePageForReplacement();
            absPtr = seg.absolute(relPtr);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            // We can clear dirty flag after the page has been allocated.
            setDirty(fullId, absPtr, false, false);
            seg.pageReplacementPolicy.onMiss(relPtr);
            seg.loadedPages.put(grpId, fullId.effectivePageId(), relPtr, seg.partGeneration(grpId, partId));
            long pageAddr = absPtr + PAGE_OVERHEAD;
            if (!restore) {
                if (delayedPageReplacementTracker != null)
                    delayedPageReplacementTracker.waitUnlock(fullId);
                readPageFromStore = true;
            } else {
                GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte) 0);
                // Must init page ID in order to ensure RWLock tag consistency.
                PageIO.setPageId(pageAddr, pageId);
            }
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
            if (readPageFromStore) {
                boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
                assert locked : "Page ID " + fullId + " expected to be locked";
                lockedPageAbsPtr = absPtr;
            }
        } else if (relPtr == OUTDATED_REL_PTR) {
            assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
            relPtr = seg.refreshOutdatedPage(grpId, pageId, false);
            absPtr = seg.absolute(relPtr);
            long pageAddr = absPtr + PAGE_OVERHEAD;
            GridUnsafe.setMemory(pageAddr, pageSize(), (byte) 0);
            PageHeader.fullPageId(absPtr, fullId);
            PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
            PageIO.setPageId(pageAddr, pageId);
            assert !PageHeader.isAcquired(absPtr) : "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) + ", absPtr=" + U.hexLong(absPtr) + ']';
            rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
            seg.pageReplacementPolicy.onRemove(relPtr);
            seg.pageReplacementPolicy.onMiss(relPtr);
        } else {
            absPtr = seg.absolute(relPtr);
            seg.pageReplacementPolicy.onHit(relPtr);
        }
        seg.acquirePage(absPtr);
        if (!readPageFromStore)
            statHolder.trackLogicalRead(absPtr + PAGE_OVERHEAD);
        return absPtr;
    } catch (IgniteOutOfMemoryException oom) {
        ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
        throw oom;
    } finally {
        seg.writeLock().unlock();
        if (delayedPageReplacementTracker != null)
            delayedPageReplacementTracker.delayedPageWrite().finishReplacement();
        if (readPageFromStore) {
            assert lockedPageAbsPtr != -1 : "Page is expected to have a valid address [pageId=" + fullId + ", lockedPageAbsPtr=" + U.hexLong(lockedPageAbsPtr) + ']';
            assert isPageWriteLocked(lockedPageAbsPtr) : "Page is expected to be locked: [pageId=" + fullId + "]";
            long pageAddr = lockedPageAbsPtr + PAGE_OVERHEAD;
            ByteBuffer buf = wrapPointer(pageAddr, pageSize());
            long actualPageId = 0;
            try {
                pmPageMgr.read(grpId, pageId, buf, false);
                statHolder.trackPhysicalAndLogicalRead(pageAddr);
                actualPageId = PageIO.getPageId(buf);
                dataRegionMetrics.onPageRead();
                if (PageIO.isIndexPage(PageIO.getType(buf)))
                    dataRegionMetrics.cacheGrpPageMetrics(grpId).indexPages().increment();
            } catch (IgniteDataIntegrityViolationException e) {
                U.warn(log, "Failed to read page (data integrity violation encountered, will try to " + "restore using existing WAL) [fullPageId=" + fullId + ']', e);
                buf.rewind();
                tryToRestorePage(fullId, buf);
                // Mark the page as dirty because it has been restored.
                setDirty(fullId, lockedPageAbsPtr, true, false);
                // And save the page snapshot in the WAL.
                beforeReleaseWrite(fullId, pageAddr, true);
                statHolder.trackPhysicalAndLogicalRead(pageAddr);
                dataRegionMetrics.onPageRead();
            } finally {
                rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, actualPageId == 0 ? OffheapReadWriteLock.TAG_LOCK_ALWAYS : PageIdUtils.tag(actualPageId));
            }
        }
    }
}

Example 3 with IgniteOutOfMemoryException

use of org.apache.ignite.internal.mem.IgniteOutOfMemoryException in project ignite by apache.

the class CacheDataRegionConfigurationTest method testTooSmallDataRegion.

/**
 * Verifies that {@link IgniteOutOfMemoryException} is thrown when cache is configured with too small DataRegion.
 */
@Test
public void testTooSmallDataRegion() throws Exception {
    memCfg = new DataStorageConfiguration();
    DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration();
    dfltPlcCfg.setName("dfltPlc");
    dfltPlcCfg.setInitialSize(DFLT_MEM_PLC_SIZE);
    dfltPlcCfg.setMaxSize(DFLT_MEM_PLC_SIZE);
    DataRegionConfiguration bigPlcCfg = new DataRegionConfiguration();
    bigPlcCfg.setName("bigPlc");
    bigPlcCfg.setMaxSize(BIG_MEM_PLC_SIZE);
    memCfg.setDataRegionConfigurations(bigPlcCfg);
    memCfg.setDefaultDataRegionConfiguration(dfltPlcCfg);
    ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
    IgniteEx ignite0 = startGrid(0);
    IgniteCache<Object, Object> cache = ignite0.cache(DEFAULT_CACHE_NAME);
    boolean oomeThrown = false;
    try {
        for (int i = 0; i < 500_000; i++) cache.put(i, "abc");
    } catch (Exception e) {
        Throwable cause = e;
        do {
            if (cause instanceof IgniteOutOfMemoryException) {
                oomeThrown = true;
                break;
            }
            if (cause == null)
                break;
            if (cause.getSuppressed() == null || cause.getSuppressed().length == 0)
                cause = cause.getCause();
            else
                cause = cause.getSuppressed()[0];
        } while (true);
    }
    if (!oomeThrown)
        fail("OutOfMemoryException hasn't been thrown");
}

Example 4 with IgniteOutOfMemoryException

use of org.apache.ignite.internal.mem.IgniteOutOfMemoryException in project ignite by apache.

the class PageMemoryNoStoreImpl method allocatePage.

/**
 * {@inheritDoc}
 */
@Override
public long allocatePage(int grpId, int partId, byte flags) {
    assert started;
    long relPtr = borrowFreePage(grpId);
    long absPtr = 0;
    if (relPtr != INVALID_REL_PTR) {
        int pageIdx = PageIdUtils.pageIndex(relPtr);
        Segment seg = segment(pageIdx);
        absPtr = seg.absolute(pageIdx);
    } else {
        // No segments contained a free page.
        Segment[] seg0 = segments;
        Segment allocSeg = seg0[seg0.length - 1];
        while (allocSeg != null) {
            relPtr = allocSeg.allocateFreePage(flags);
            if (relPtr != INVALID_REL_PTR) {
                absPtr = allocSeg.absolute(PageIdUtils.pageIndex(relPtr));
                allocatedPages.incrementAndGet();
                PageMetrics grpPageMetrics = dataRegionMetrics.cacheGrpPageMetrics(grpId);
                grpPageMetrics.totalPages().increment();
                break;
            } else
                allocSeg = addSegment(seg0);
        }
    }
    if (relPtr == INVALID_REL_PTR) {
        IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + "name=" + dataRegionCfg.getName() + ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) + ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) + ", persistenceEnabled=" + dataRegionCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + "  ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + "  ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + "  ^-- Enable eviction or expiration policies");
        if (ctx != null)
            ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
        throw oom;
    }
    assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & ~PageIdUtils.PAGE_IDX_MASK);
    // Assign page ID according to flags and partition ID.
    long pageId = PageIdUtils.pageId(partId, flags, (int) relPtr);
    writePageId(absPtr, pageId);
    // TODO pass an argument to decide whether the page should be cleaned.
    GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, sysPageSize - PAGE_OVERHEAD, (byte) 0);
    return pageId;
}

Example 5 with IgniteOutOfMemoryException

use of org.apache.ignite.internal.mem.IgniteOutOfMemoryException in project ignite by apache.

the class PageMemoryImplTest method testCheckpointBufferOverusageDontCauseWriteLockLeak.

/**
 * @throws Exception If failed.
 */
@Test
public void testCheckpointBufferOverusageDontCauseWriteLockLeak() throws Exception {
    PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED, null);
    List<FullPageId> pages = new ArrayList<>();
    try {
        while (!Thread.currentThread().isInterrupted()) {
            long pageId = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
            FullPageId fullPageId = new FullPageId(pageId, 1);
            pages.add(fullPageId);
            // to set page id, otherwise we would fail with assertion error
            acquireAndReleaseWriteLock(memory, fullPageId);
        }
    } catch (IgniteOutOfMemoryException ignore) {
    // Success
    }
    memory.beginCheckpoint(new GridFinishedFuture());
    final AtomicReference<FullPageId> lastPage = new AtomicReference<>();
    try {
        for (FullPageId fullPageId : pages) {
            lastPage.set(fullPageId);
            acquireAndReleaseWriteLock(memory, fullPageId);
        }
    } catch (Exception ex) {
        assertTrue(ex.getMessage().startsWith(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG));
    }
    memory.finishCheckpoint();
    GridTestUtils.runAsync(() -> {
        try {
            // we should be able get lock again
            acquireAndReleaseWriteLock(memory, lastPage.get());
        } catch (IgniteCheckedException e) {
            throw new AssertionError(e);
        }
    }).get(getTestTimeout());
}