Examples with PageMemoryEx org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx used on opensource projects

Example 6 with PageMemoryEx

use of org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx in project ignite by apache.

the class GridCacheOffheapManager method saveStoreMetadata.

/**
 * @param store Store to save metadata.
 * @throws IgniteCheckedException If failed.
 */
private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean saveMeta, boolean beforeDestroy) throws IgniteCheckedException {
    RowStore rowStore0 = store.rowStore();
    boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
    boolean wasSaveToMeta = false;
    if (rowStore0 != null) {
        CacheFreeListImpl freeList = (CacheFreeListImpl) rowStore0.freeList();
        freeList.saveMetadata();
        long updCntr = store.updateCounter();
        int size = store.fullSize();
        long rmvId = globalRemoveId().get();
        PageMemoryEx pageMem = (PageMemoryEx) grp.dataRegion().pageMemory();
        IgniteWriteAheadLogManager wal = this.ctx.wal();
        if (size > 0 || updCntr > 0) {
            GridDhtPartitionState state = null;
            // localPartition will not acquire writeLock here because create=false.
            GridDhtLocalPartition part = null;
            if (!grp.isLocal()) {
                if (beforeDestroy)
                    state = GridDhtPartitionState.EVICTED;
                else {
                    part = getPartition(store);
                    if (part != null && part.state() != GridDhtPartitionState.EVICTED)
                        state = part.state();
                }
                // Do not save meta for evicted partitions on next checkpoints.
                if (state == null)
                    return false;
            }
            int grpId = grp.groupId();
            long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId());
            long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
            try {
                long partMetaPageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
                if (partMetaPageAddr == 0L) {
                    U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size + ", updCntr=" + updCntr + ", state=" + state + ']');
                    return false;
                }
                boolean changed = false;
                try {
                    PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
                    changed |= io.setUpdateCounter(partMetaPageAddr, updCntr);
                    changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId);
                    changed |= io.setSize(partMetaPageAddr, size);
                    if (state != null)
                        changed |= io.setPartitionState(partMetaPageAddr, (byte) state.ordinal());
                    else
                        assert grp.isLocal() : grp.cacheOrGroupName();
                    long cntrsPageId;
                    if (grp.sharedGroup()) {
                        long initCntrPageId = io.getCountersPageId(partMetaPageAddr);
                        Map<Integer, Long> newSizes = store.cacheSizes();
                        Map<Integer, Long> prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId);
                        if (prevSizes != null && prevSizes.equals(newSizes))
                            // Preventing modification of sizes pages for store
                            cntrsPageId = initCntrPageId;
                        else {
                            cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId, store.partId(), newSizes);
                            if (initCntrPageId == 0 && cntrsPageId != 0) {
                                io.setCountersPageId(partMetaPageAddr, cntrsPageId);
                                changed = true;
                            }
                        }
                    } else
                        cntrsPageId = 0L;
                    int pageCnt;
                    if (needSnapshot) {
                        pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
                        io.setCandidatePageCount(partMetaPageAddr, size == 0 ? 0 : pageCnt);
                        if (saveMeta) {
                            saveMeta(ctx);
                            wasSaveToMeta = true;
                        }
                        if (state == OWNING) {
                            assert part != null;
                            if (!addPartition(part, ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(), this.ctx.pageStore().pages(grpId, store.partId()), store.fullSize()))
                                U.warn(log, "Partition was concurrently evicted grpId=" + grpId + ", partitionId=" + part.id());
                        } else if (state == MOVING || state == RENTING) {
                            if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) {
                                if (log.isInfoEnabled())
                                    log.info("Will not include SQL indexes to snapshot because there is " + "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() + ", partId=" + store.partId() + ", state=" + state + ']');
                            }
                        }
                        changed = true;
                    } else
                        pageCnt = io.getCandidatePageCount(partMetaPageAddr);
                    if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
                        wal.log(new MetaPageUpdatePartitionDataRecord(grpId, partMetaId, updCntr, rmvId, size, cntrsPageId, state == null ? -1 : (byte) state.ordinal(), pageCnt));
                } finally {
                    pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
                }
            } finally {
                pageMem.releasePage(grpId, partMetaId, partMetaPage);
            }
        } else if (needSnapshot)
            tryAddEmptyPartitionToSnapshot(store, ctx);
        ;
    } else if (needSnapshot)
        tryAddEmptyPartitionToSnapshot(store, ctx);
    return wasSaveToMeta;
}

Example 7 with PageMemoryEx

use of org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx in project ignite by apache.

the class GridCacheDatabaseSharedManager method applyLastUpdates.

/**
 * @param status Last registered checkpoint status.
 * @throws IgniteCheckedException If failed to apply updates.
 * @throws StorageException If IO exception occurred while reading write-ahead log.
 */
private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException {
    if (log.isInfoEnabled())
        log.info("Applying lost cache updates since last checkpoint record [lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
    if (!metastoreOnly)
        cctx.kernalContext().query().skipFieldLookup(true);
    long start = U.currentTimeMillis();
    int applied = 0;
    Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
    try (WALIterator it = cctx.wal().replay(status.startPtr)) {
        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
        while (it.hasNextX()) {
            IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
            WALRecord rec = next.get2();
            switch(rec.type()) {
                case DATA_RECORD:
                    if (metastoreOnly)
                        continue;
                    DataRecord dataRec = (DataRecord) rec;
                    for (DataEntry dataEntry : dataRec.writeEntries()) {
                        int cacheId = dataEntry.cacheId();
                        int grpId = cctx.cache().cacheDescriptor(cacheId).groupId();
                        if (!ignoreGrps.contains(grpId)) {
                            GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
                            applyUpdate(cacheCtx, dataEntry);
                            applied++;
                        }
                    }
                    break;
                case PART_META_UPDATE_STATE:
                    if (metastoreOnly)
                        continue;
                    PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord) rec;
                    if (!ignoreGrps.contains(metaStateRecord.groupId())) {
                        partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()), new T2<>((int) metaStateRecord.state(), metaStateRecord.updateCounter()));
                    }
                    break;
                case METASTORE_DATA_RECORD:
                    MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord) rec;
                    metaStorage.applyUpdate(metastoreDataRecord.key(), metastoreDataRecord.value());
                    break;
                case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
                case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
                case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
                    if (metastoreOnly)
                        continue;
                    PageDeltaRecord rec0 = (PageDeltaRecord) rec;
                    PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId());
                    long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true);
                    try {
                        long addr = pageMem.writeLock(rec0.groupId(), rec0.pageId(), page, true);
                        try {
                            rec0.applyDelta(pageMem, addr);
                        } finally {
                            pageMem.writeUnlock(rec0.groupId(), rec0.pageId(), page, null, true, true);
                        }
                    } finally {
                        pageMem.releasePage(rec0.groupId(), rec0.pageId(), page);
                    }
                    break;
                default:
            }
        }
        if (!metastoreOnly)
            restorePartitionState(partStates, ignoreGrps);
    } finally {
        if (!metastoreOnly)
            cctx.kernalContext().query().skipFieldLookup(false);
    }
    if (log.isInfoEnabled())
        log.info("Finished applying WAL changes [updatesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + "ms]");
}

Example 8 with PageMemoryEx

use of org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx in project ignite by apache.

the class GridCacheDatabaseSharedManager method onCacheGroupsStopped.

/**
 * {@inheritDoc}
 */
@Override
public void onCacheGroupsStopped(Collection<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGrps) {
    Map<PageMemoryEx, Collection<Integer>> destroyed = new HashMap<>();
    for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
        CacheGroupContext gctx = tup.get1();
        if (!gctx.persistenceEnabled())
            continue;
        snapshotMgr.onCacheGroupStop(gctx);
        PageMemoryEx pageMem = (PageMemoryEx) gctx.dataRegion().pageMemory();
        Collection<Integer> grpIds = destroyed.get(pageMem);
        if (grpIds == null) {
            grpIds = new HashSet<>();
            destroyed.put(pageMem, grpIds);
        }
        grpIds.add(tup.get1().groupId());
        pageMem.onCacheGroupDestroyed(tup.get1().groupId());
    }
    Collection<IgniteInternalFuture<Void>> clearFuts = new ArrayList<>(destroyed.size());
    for (Map.Entry<PageMemoryEx, Collection<Integer>> entry : destroyed.entrySet()) {
        final Collection<Integer> grpIds = entry.getValue();
        clearFuts.add(entry.getKey().clearAsync((grpId, pageIdg) -> grpIds.contains(grpId), false));
    }
    for (IgniteInternalFuture<Void> clearFut : clearFuts) {
        try {
            clearFut.get();
        } catch (IgniteCheckedException e) {
            log.error("Failed to clear page memory", e);
        }
    }
    if (cctx.pageStore() != null) {
        for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
            CacheGroupContext grp = tup.get1();
            if (grp.affinityNode()) {
                try {
                    cctx.pageStore().shutdownForCacheGroup(grp, tup.get2());
                } catch (IgniteCheckedException e) {
                    U.error(log, "Failed to gracefully clean page store resources for destroyed cache " + "[cache=" + grp.cacheOrGroupName() + "]", e);
                }
            }
        }
    }
}

Example 9 with PageMemoryEx

use of org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx in project ignite by apache.

the class GridCacheDatabaseSharedManager method readMetastore.

/**
 */
private void readMetastore() throws IgniteCheckedException {
    try {
        DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
        DataRegionConfiguration plcCfg = createDataRegionConfiguration(memCfg);
        File allocPath = buildAllocPath(plcCfg);
        DirectMemoryProvider memProvider = allocPath == null ? new UnsafeMemoryProvider(log) : new MappedFileMemoryProvider(log, allocPath);
        DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(plcCfg);
        PageMemoryEx storePageMem = (PageMemoryEx) createPageMemory(memProvider, memCfg, plcCfg, memMetrics, false);
        DataRegion regCfg = new DataRegion(storePageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, storePageMem));
        CheckpointStatus status = readCheckpointStatus();
        cctx.pageStore().initializeForMetastorage();
        storePageMem.start();
        checkpointReadLock();
        try {
            restoreMemory(status, true, storePageMem);
            metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
            metaStorage.init(this);
            applyLastUpdates(status, true);
            initiallyWalDisabledGrps = walDisabledGroups();
            notifyMetastorageReadyForRead();
        } finally {
            checkpointReadUnlock();
        }
        metaStorage = null;
        storePageMem.stop();
    } catch (StorageException e) {
        throw new IgniteCheckedException(e);
    }
}

Example 10 with PageMemoryEx

use of org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx in project ignite by apache.

the class GridCacheDatabaseSharedManager method readPartitionState.

/**
 * @param grpCtx Group context.
 * @param partId Partition ID.
 * @return Partition state.
 */
public GridDhtPartitionState readPartitionState(CacheGroupContext grpCtx, int partId) {
    int grpId = grpCtx.groupId();
    PageMemoryEx pageMem = (PageMemoryEx) grpCtx.dataRegion().pageMemory();
    try {
        if (storeMgr.exists(grpId, partId)) {
            storeMgr.ensure(grpId, partId);
            if (storeMgr.pages(grpId, partId) > 1) {
                long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
                long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
                try {
                    long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
                    try {
                        if (PageIO.getType(pageAddr) == PageIO.T_PART_META) {
                            PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
                            GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal((int) io.getPartitionState(pageAddr));
                            if (state == null)
                                state = GridDhtPartitionState.MOVING;
                            return state;
                        }
                    } finally {
                        pageMem.readUnlock(grpId, partMetaId, partMetaPage);
                    }
                } finally {
                    pageMem.releasePage(grpId, partMetaId, partMetaPage);
                }
            }
        }
    } catch (IgniteCheckedException e) {
        U.error(log, "Failed to read partition state (will default to MOVING) [grp=" + grpCtx + ", partId=" + partId + "]", e);
    }
    return GridDhtPartitionState.MOVING;
}