Examples with CacheFreeList org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList used on opensource projects

Example 1 with CacheFreeList

use of org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList in project ignite by apache.

the class GridCacheOffheapManager method saveStoreMetadata.

/**
 * @param store Store to save metadata.
 * @throws IgniteCheckedException If failed.
 */
private void saveStoreMetadata(CacheDataStore store, Context ctx, boolean beforeDestroy, boolean needSnapshot) throws IgniteCheckedException {
    RowStore rowStore0 = store.rowStore();
    if (rowStore0 != null && (partitionStatesRestored || grp.isLocal())) {
        ((CacheFreeList) rowStore0.freeList()).saveMetadata(grp.statisticsHolderData());
        PartitionMetaStorage<SimpleDataRow> partStore = store.partStorage();
        long updCntr = store.updateCounter();
        long size = store.fullSize();
        long rmvId = globalRemoveId().get();
        byte[] updCntrsBytes = store.partUpdateCounter().getBytes();
        PageMemoryEx pageMem = (PageMemoryEx) grp.dataRegion().pageMemory();
        IgniteWriteAheadLogManager wal = this.ctx.wal();
        GridEncryptionManager encMgr = this.ctx.kernalContext().encryption();
        if (size > 0 || updCntr > 0 || !store.partUpdateCounter().sequential() || (grp.config().isEncryptionEnabled() && encMgr.getEncryptionState(grp.groupId(), store.partId()) > 0)) {
            GridDhtPartitionState state = null;
            // localPartition will not acquire writeLock here because create=false.
            GridDhtLocalPartition part = null;
            if (!grp.isLocal()) {
                if (beforeDestroy)
                    state = GridDhtPartitionState.EVICTED;
                else {
                    part = getPartition(store);
                    if (part != null && part.state() != GridDhtPartitionState.EVICTED)
                        state = part.state();
                }
                // Do not save meta for evicted partitions on next checkpoints.
                if (state == null)
                    return;
            }
            int grpId = grp.groupId();
            long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId());
            long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
            try {
                long partMetaPageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
                if (partMetaPageAddr == 0L) {
                    U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ", beforeDestroy=" + beforeDestroy + ", size=" + size + ", updCntr=" + updCntr + ", state=" + state + ']');
                    return;
                }
                boolean changed = false;
                try {
                    PagePartitionMetaIOV3 io = PageIO.getPageIO(partMetaPageAddr);
                    long link = io.getGapsLink(partMetaPageAddr);
                    if (updCntrsBytes == null && link != 0) {
                        partStore.removeDataRowByLink(link, grp.statisticsHolderData());
                        io.setGapsLink(partMetaPageAddr, (link = 0));
                        changed = true;
                    } else if (updCntrsBytes != null && link == 0) {
                        SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes);
                        partStore.insertDataRow(row, grp.statisticsHolderData());
                        io.setGapsLink(partMetaPageAddr, (link = row.link()));
                        changed = true;
                    } else if (updCntrsBytes != null && link != 0) {
                        byte[] prev = partStore.readRow(link);
                        assert prev != null : "Read null gaps using link=" + link;
                        if (!Arrays.equals(prev, updCntrsBytes)) {
                            partStore.removeDataRowByLink(link, grp.statisticsHolderData());
                            SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes);
                            partStore.insertDataRow(row, grp.statisticsHolderData());
                            io.setGapsLink(partMetaPageAddr, (link = row.link()));
                            changed = true;
                        }
                    }
                    if (changed)
                        partStore.saveMetadata(grp.statisticsHolderData());
                    changed |= io.setUpdateCounter(partMetaPageAddr, updCntr);
                    changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId);
                    changed |= io.setSize(partMetaPageAddr, size);
                    int encryptIdx = 0;
                    int encryptCnt = 0;
                    if (grp.config().isEncryptionEnabled()) {
                        long reencryptState = encMgr.getEncryptionState(grpId, store.partId());
                        if (reencryptState != 0) {
                            encryptIdx = ReencryptStateUtils.pageIndex(reencryptState);
                            encryptCnt = ReencryptStateUtils.pageCount(reencryptState);
                            if (encryptIdx == encryptCnt) {
                                encMgr.setEncryptionState(grp, store.partId(), 0, 0);
                                encryptIdx = encryptCnt = 0;
                            }
                            changed |= io.setEncryptedPageIndex(partMetaPageAddr, encryptIdx);
                            changed |= io.setEncryptedPageCount(partMetaPageAddr, encryptCnt);
                        }
                    }
                    if (state != null)
                        changed |= io.setPartitionState(partMetaPageAddr, (byte) state.ordinal());
                    else
                        assert grp.isLocal() : grp.cacheOrGroupName();
                    long cntrsPageId;
                    if (grp.sharedGroup()) {
                        long initCntrPageId = io.getCountersPageId(partMetaPageAddr);
                        Map<Integer, Long> newSizes = store.cacheSizes();
                        Map<Integer, Long> prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId);
                        if (prevSizes != null && prevSizes.equals(newSizes))
                            // Preventing modification of sizes pages for store
                            cntrsPageId = initCntrPageId;
                        else {
                            cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId, store.partId(), newSizes);
                            if (initCntrPageId == 0 && cntrsPageId != 0) {
                                io.setCountersPageId(partMetaPageAddr, cntrsPageId);
                                changed = true;
                            }
                        }
                    } else
                        cntrsPageId = 0L;
                    int pageCnt;
                    if (needSnapshot) {
                        pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
                        io.setCandidatePageCount(partMetaPageAddr, size == 0 ? 0 : pageCnt);
                        if (state == OWNING) {
                            assert part != null;
                            if (!addPartition(part, ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(), this.ctx.pageStore().pages(grpId, store.partId()), store.fullSize()))
                                U.warn(log, "Partition was concurrently evicted grpId=" + grpId + ", partitionId=" + part.id());
                        } else if (state == MOVING || state == RENTING) {
                            if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) {
                                if (log.isInfoEnabled())
                                    log.info("Will not include SQL indexes to snapshot because there is " + "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() + ", partId=" + store.partId() + ", state=" + state + ']');
                            }
                        }
                        changed = true;
                    } else
                        pageCnt = io.getCandidatePageCount(partMetaPageAddr);
                    if (changed && isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
                        wal.log(new MetaPageUpdatePartitionDataRecordV3(grpId, partMetaId, updCntr, rmvId, // TODO: Partition size may be long
                        (int) size, cntrsPageId, state == null ? -1 : (byte) state.ordinal(), pageCnt, link, encryptIdx, encryptCnt));
                    if (changed) {
                        partStore.saveMetadata(grp.statisticsHolderData());
                        io.setPartitionMetaStoreReuseListRoot(partMetaPageAddr, partStore.metaPageId());
                    }
                } finally {
                    pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
                }
            } finally {
                pageMem.releasePage(grpId, partMetaId, partMetaPage);
            }
        } else if (needSnapshot)
            tryAddEmptyPartitionToSnapshot(store, ctx);
    } else if (needSnapshot)
        tryAddEmptyPartitionToSnapshot(store, ctx);
}

Example 2 with CacheFreeList

use of org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList in project ignite by apache.

the class IgniteCacheDatabaseSharedManager method start0.

/**
 * {@inheritDoc}
 */
@Override
protected void start0() throws IgniteCheckedException {
    if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null)
        return;
    DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
    assert memCfg != null;
    validateConfiguration(memCfg);
    pageSize = memCfg.getPageSize();
    initDataRegions(memCfg);
    cctx.kernalContext().systemView().registerView(DATA_REGION_PAGE_LIST_VIEW, DATA_REGION_PAGE_LIST_VIEW_DESC, new PagesListViewWalker(), () -> {
        Map<String, CacheFreeList> freeLists = freeListMap;
        if (freeLists == null)
            return Collections.emptyList();
        return freeLists.values().stream().flatMap(fl -> IntStream.range(0, fl.bucketsCount()).mapToObj(bucket -> new PagesListView(fl, bucket))).collect(Collectors.toList());
    }, Function.identity());
}

Example 3 with CacheFreeList

use of org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList in project ignite by apache.

the class IgniteCacheDatabaseSharedManager method freeSpaceProvider.

/**
 * Closure that can be used to compute fill factor for provided data region.
 *
 * @param dataRegCfg Data region configuration.
 * @return Closure.
 *
 * @deprecated use {@link #dataRegionMetricsProvider(DataRegionConfiguration)} instead.
 */
@Deprecated
protected IgniteOutClosure<Long> freeSpaceProvider(final DataRegionConfiguration dataRegCfg) {
    final String dataRegName = dataRegCfg.getName();
    return new IgniteOutClosure<Long>() {

        private CacheFreeList freeList;

        @Override
        public Long apply() {
            if (freeList == null) {
                CacheFreeList freeList0 = freeListMap.get(dataRegName);
                if (freeList0 == null)
                    return 0L;
                freeList = freeList0;
            }
            return freeList.freeSpace();
        }
    };
}

Example 4 with CacheFreeList

use of org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList in project ignite by apache.

the class IgniteCacheDatabaseSharedManager method ensureFreeSpaceForInsert.

/**
 * Checks that the given {@code region} has enough space for putting a new entry.
 *
 * This method makes sense then and only then
 * the data region is not persisted {@link DataRegionConfiguration#isPersistenceEnabled()}
 * and page eviction is disabled {@link DataPageEvictionMode#DISABLED}.
 *
 * The non-persistent region should reserve a number of pages to support a free list {@link AbstractFreeList}.
 * For example, removing a row from underlying store may require allocating a new data page
 * in order to move a tracked page from one bucket to another one which does not have a free space for a new stripe.
 * See {@link AbstractFreeList#removeDataRowByLink}.
 * Therefore, inserting a new entry should be prevented in case of some threshold is exceeded.
 *
 * @param region Data region to be checked.
 * @param dataRowSize Size of data row to be inserted.
 * @throws IgniteOutOfMemoryException In case of the given data region does not have enough free space
 * for putting a new entry.
 */
public void ensureFreeSpaceForInsert(DataRegion region, int dataRowSize) throws IgniteOutOfMemoryException {
    if (region == null)
        return;
    DataRegionConfiguration regCfg = region.config();
    if (regCfg.getPageEvictionMode() != DataPageEvictionMode.DISABLED || regCfg.isPersistenceEnabled())
        return;
    long memorySize = regCfg.getMaxSize();
    PageMemory pageMem = region.pageMemory();
    CacheFreeList freeList = freeListMap.get(regCfg.getName());
    long nonEmptyPages = (pageMem.loadedPages() - freeList.emptyDataPages());
    // The maximum number of pages that can be allocated (memorySize / systemPageSize)
    // should be greater or equal to pages required for inserting a new entry plus
    // the current number of non-empty pages plus the number of pages that may be required in order to move
    // all pages to a reuse bucket, that is equal to nonEmptyPages * 8 / pageSize, where 8 is the size of a link.
    // Note that not the whole page can be used to storing links,
    // see PagesListNodeIO and PagesListMetaIO#getCapacity(), so we pessimistically multiply the result on 1.5,
    // in any way, the number of required pages is less than 1 percent.
    boolean oomThreshold = (memorySize / pageMem.systemPageSize()) < ((double) dataRowSize / pageMem.pageSize() + nonEmptyPages * (8.0 * 1.5 / pageMem.pageSize() + 1) + 256);
    if (oomThreshold) {
        IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + "name=" + regCfg.getName() + ", initSize=" + U.readableSize(regCfg.getInitialSize(), false) + ", maxSize=" + U.readableSize(regCfg.getMaxSize(), false) + ", persistenceEnabled=" + regCfg.isPersistenceEnabled() + "] Try the following:" + U.nl() + "  ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + "  ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + "  ^-- Enable eviction or expiration policies");
        if (cctx.kernalContext() != null)
            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
        throw oom;
    }
}

Example 5 with CacheFreeList

use of org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeList in project ignite by apache.

the class IgniteCacheDatabaseSharedManager method initPageMemoryDataStructures.

/**
 * @param dbCfg Database config.
 * @throws IgniteCheckedException If failed.
 */
protected void initPageMemoryDataStructures(DataStorageConfiguration dbCfg) throws IgniteCheckedException {
    freeListMap = U.newHashMap(dataRegionMap.size());
    String dfltMemPlcName = dbCfg.getDefaultDataRegionConfiguration().getName();
    for (DataRegion memPlc : dataRegionMap.values()) {
        DataRegionConfiguration memPlcCfg = memPlc.config();
        boolean persistenceEnabled = memPlcCfg.isPersistenceEnabled();
        String freeListName = memPlcCfg.getName() + "##FreeList";
        CacheFreeList freeList = new CacheFreeList(0, freeListName, memPlc, persistenceEnabled ? cctx.wal() : null, 0L, true, cctx.diagnostic().pageLockTracker(), cctx.kernalContext(), null, PageIdAllocator.FLAG_IDX);
        freeListMap.put(memPlcCfg.getName(), freeList);
    }
    dfltFreeList = freeListMap.get(dfltMemPlcName);
}