Search in sources :

Example 1 with DataPageEvictionMode

use of org.apache.ignite.configuration.DataPageEvictionMode in project ignite by apache.

the class CachePartitionDefragmentationManager method executeDefragmentation.

/**
 */
public void executeDefragmentation() throws IgniteCheckedException {
    Map<Integer, List<CacheDataStore>> oldStores = new HashMap<>();
    for (CacheGroupContext oldGrpCtx : cacheGrpCtxsForDefragmentation) {
        int grpId = oldGrpCtx.groupId();
        final IgniteCacheOffheapManager offheap = oldGrpCtx.offheap();
        List<CacheDataStore> oldCacheDataStores = stream(offheap.cacheDataStores().spliterator(), false).filter(store -> {
            try {
                return filePageStoreMgr.exists(grpId, store.partId());
            } catch (IgniteCheckedException e) {
                throw new IgniteException(e);
            }
        }).collect(Collectors.toList());
        oldStores.put(grpId, oldCacheDataStores);
    }
    int partitionCount = oldStores.values().stream().mapToInt(List::size).sum();
    status.onStart(cacheGrpCtxsForDefragmentation, partitionCount);
    try {
        // Now the actual process starts.
        IgniteInternalFuture<?> idxDfrgFut = null;
        DataPageEvictionMode prevPageEvictionMode = null;
        for (CacheGroupContext oldGrpCtx : cacheGrpCtxsForDefragmentation) {
            int grpId = oldGrpCtx.groupId();
            File workDir = filePageStoreMgr.cacheWorkDir(oldGrpCtx.sharedGroup(), oldGrpCtx.cacheOrGroupName());
            List<CacheDataStore> oldCacheDataStores = oldStores.get(grpId);
            if (skipAlreadyDefragmentedCacheGroup(workDir, grpId, log)) {
                status.onCacheGroupSkipped(oldGrpCtx, oldCacheDataStores.size());
                continue;
            }
            try {
                GridCacheOffheapManager offheap = (GridCacheOffheapManager) oldGrpCtx.offheap();
                status.onCacheGroupStart(oldGrpCtx, oldCacheDataStores.size());
                if (workDir == null || oldCacheDataStores.isEmpty()) {
                    status.onCacheGroupFinish(oldGrpCtx);
                    continue;
                }
                // We can't start defragmentation of new group on the region that has wrong eviction mode.
                // So waiting of the previous cache group defragmentation is inevitable.
                DataPageEvictionMode curPageEvictionMode = oldGrpCtx.dataRegion().config().getPageEvictionMode();
                if (prevPageEvictionMode == null || prevPageEvictionMode != curPageEvictionMode) {
                    prevPageEvictionMode = curPageEvictionMode;
                    partDataRegion.config().setPageEvictionMode(curPageEvictionMode);
                    if (idxDfrgFut != null)
                        idxDfrgFut.get();
                }
                IntMap<CacheDataStore> cacheDataStores = new IntHashMap<>();
                for (CacheDataStore store : offheap.cacheDataStores()) {
                    // This would mean that these partitions are empty.
                    assert store.tree() == null || store.tree().groupId() == grpId;
                    if (store.tree() != null)
                        cacheDataStores.put(store.partId(), store);
                }
                dbMgr.checkpointedDataRegions().remove(oldGrpCtx.dataRegion());
                // Another cheat. Ttl cleanup manager knows too much shit.
                oldGrpCtx.caches().stream().filter(cacheCtx -> cacheCtx.groupId() == grpId).forEach(cacheCtx -> cacheCtx.ttl().unregister());
                // Technically wal is already disabled, but "PageHandler.isWalDeltaRecordNeeded" doesn't care
                // and WAL records will be allocated anyway just to be ignored later if we don't disable WAL for
                // cache group explicitly.
                oldGrpCtx.localWalEnabled(false, false);
                boolean encrypted = oldGrpCtx.config().isEncryptionEnabled();
                FileVersionCheckingFactory pageStoreFactory = filePageStoreMgr.getPageStoreFactory(grpId, encrypted);
                AtomicLong idxAllocationTracker = new GridAtomicLong();
                createIndexPageStore(grpId, workDir, pageStoreFactory, partDataRegion, idxAllocationTracker::addAndGet);
                checkCancellation();
                GridCompoundFuture<Object, Object> cmpFut = new GridCompoundFuture<>();
                PageMemoryEx oldPageMem = (PageMemoryEx) oldGrpCtx.dataRegion().pageMemory();
                CacheGroupContext newGrpCtx = new CacheGroupContext(sharedCtx, grpId, oldGrpCtx.receivedFrom(), CacheType.USER, oldGrpCtx.config(), oldGrpCtx.affinityNode(), partDataRegion, oldGrpCtx.cacheObjectContext(), null, null, oldGrpCtx.localStartVersion(), true, false, true);
                defragmentationCheckpoint.checkpointTimeoutLock().checkpointReadLock();
                try {
                    // This will initialize partition meta in index partition - meta tree and reuse list.
                    newGrpCtx.start();
                } finally {
                    defragmentationCheckpoint.checkpointTimeoutLock().checkpointReadUnlock();
                }
                IgniteUtils.doInParallel(defragmentationThreadPool, oldCacheDataStores, oldCacheDataStore -> defragmentOnePartition(oldGrpCtx, grpId, workDir, offheap, pageStoreFactory, cmpFut, oldPageMem, newGrpCtx, oldCacheDataStore));
                // A bit too general for now, but I like it more then saving only the last checkpoint future.
                cmpFut.markInitialized().get();
                idxDfrgFut = new GridFinishedFuture<>();
                if (filePageStoreMgr.hasIndexStore(grpId)) {
                    defragmentIndexPartition(oldGrpCtx, newGrpCtx);
                    idxDfrgFut = defragmentationCheckpoint.forceCheckpoint("index defragmented", null).futureFor(FINISHED);
                }
                PageStore oldIdxPageStore = filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
                idxDfrgFut = idxDfrgFut.chain(fut -> {
                    if (log.isDebugEnabled()) {
                        log.debug(S.toString("Index partition defragmented", "grpId", grpId, false, "oldPages", oldIdxPageStore.pages(), false, "newPages", idxAllocationTracker.get() + 1, false, "pageSize", pageSize, false, "partFile", defragmentedIndexFile(workDir).getName(), false, "workDir", workDir, false));
                    }
                    oldPageMem.invalidate(grpId, INDEX_PARTITION);
                    PageMemoryEx partPageMem = (PageMemoryEx) partDataRegion.pageMemory();
                    partPageMem.invalidate(grpId, INDEX_PARTITION);
                    DefragmentationPageReadWriteManager pageMgr = (DefragmentationPageReadWriteManager) partPageMem.pageManager();
                    pageMgr.pageStoreMap().removePageStore(grpId, INDEX_PARTITION);
                    PageMemoryEx mappingPageMem = (PageMemoryEx) mappingDataRegion.pageMemory();
                    pageMgr = (DefragmentationPageReadWriteManager) mappingPageMem.pageManager();
                    pageMgr.pageStoreMap().clear(grpId);
                    renameTempIndexFile(workDir);
                    writeDefragmentationCompletionMarker(filePageStoreMgr.getPageStoreFileIoFactory(), workDir, log);
                    batchRenameDefragmentedCacheGroupPartitions(workDir, log);
                    return null;
                });
                status.onIndexDefragmented(oldGrpCtx, oldIdxPageStore.size(), // + file header.
                pageSize + idxAllocationTracker.get() * pageSize);
            } catch (DefragmentationCancelledException e) {
                DefragmentationFileUtils.deleteLeftovers(workDir);
                throw e;
            }
            status.onCacheGroupFinish(oldGrpCtx);
        }
        if (idxDfrgFut != null)
            idxDfrgFut.get();
        mntcReg.unregisterMaintenanceTask(DEFRAGMENTATION_MNTC_TASK_NAME);
        status.onFinish();
        completionFut.onDone();
    } catch (DefragmentationCancelledException e) {
        mntcReg.unregisterMaintenanceTask(DEFRAGMENTATION_MNTC_TASK_NAME);
        log.info("Defragmentation process has been cancelled.");
        status.onFinish();
        completionFut.onDone();
    } catch (Throwable t) {
        log.error("Defragmentation process failed.", t);
        status.onFinish();
        completionFut.onDone(t);
        throw t;
    } finally {
        defragmentationCheckpoint.stop(true);
    }
}
Also used : IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) Arrays(java.util.Arrays) PageStore(org.apache.ignite.internal.pagemem.store.PageStore) CacheType(org.apache.ignite.internal.processors.cache.CacheType) DataRegion(org.apache.ignite.internal.processors.cache.persistence.DataRegion) GridFutureAdapter(org.apache.ignite.internal.util.future.GridFutureAdapter) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture) FLAG_DATA(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA) IntMap(org.apache.ignite.internal.util.collection.IntMap) DEFRAGMENTATION_MAPPING_REGION_NAME(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_MAPPING_REGION_NAME) DEFRAGMENTATION_PART_REGION_NAME(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_PART_REGION_NAME) DefragmentationFileUtils.defragmentedPartMappingFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartMappingFile) CacheDataStore(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore) CheckpointTimeoutLock(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock) DefragmentationFileUtils.defragmentedPartFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartFile) LightweightCheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.LightweightCheckpointManager) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MaintenanceRegistry(org.apache.ignite.maintenance.MaintenanceRegistry) Map(java.util.Map) Path(java.nio.file.Path) AbstractFreeList(org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList) IndexProcessor(org.apache.ignite.internal.cache.query.index.IndexProcessor) DataRow(org.apache.ignite.internal.processors.cache.tree.DataRow) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) IgniteOutClosure(org.apache.ignite.lang.IgniteOutClosure) DefragmentationFileUtils.skipAlreadyDefragmentedPartition(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.skipAlreadyDefragmentedPartition) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) Set(java.util.Set) PendingRow(org.apache.ignite.internal.processors.cache.tree.PendingRow) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) IgniteCacheOffheapManager(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) IntRWHashMap(org.apache.ignite.internal.util.collection.IntRWHashMap) List(java.util.List) GridCacheDataStore(org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore) StreamSupport.stream(java.util.stream.StreamSupport.stream) PageIdUtils(org.apache.ignite.internal.pagemem.PageIdUtils) CU(org.apache.ignite.internal.util.typedef.internal.CU) PagePartitionMetaIOV3(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV3) IntHashMap(org.apache.ignite.internal.util.collection.IntHashMap) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) GridAtomicLong(org.apache.ignite.internal.util.GridAtomicLong) SimpleDataRow(org.apache.ignite.internal.processors.cache.persistence.freelist.SimpleDataRow) DefragmentationFileUtils.defragmentedIndexTmpFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexTmpFile) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) GridCompoundFuture(org.apache.ignite.internal.util.future.GridCompoundFuture) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) IgniteLogger(org.apache.ignite.IgniteLogger) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) DefragmentationFileUtils.renameTempIndexFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.renameTempIndexFile) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) GridCacheOffheapManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager) CheckpointManager(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager) DefragmentationFileUtils.writeDefragmentationCompletionMarker(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.writeDefragmentationCompletionMarker) DefragmentationFileUtils.skipAlreadyDefragmentedCacheGroup(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.skipAlreadyDefragmentedCacheGroup) IgniteUtils(org.apache.ignite.internal.util.IgniteUtils) S(org.apache.ignite.internal.util.typedef.internal.S) DefragmentationFileUtils.renameTempPartitionFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.renameTempPartitionFile) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) IgniteThreadPoolExecutor(org.apache.ignite.thread.IgniteThreadPoolExecutor) Comparator.comparing(java.util.Comparator.comparing) CacheDataTree(org.apache.ignite.internal.processors.cache.tree.CacheDataTree) AbstractDataLeafIO(org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO) FINISHED(org.apache.ignite.internal.processors.cache.persistence.CheckpointState.FINISHED) PendingEntriesTree(org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree) DefragmentationFileUtils.defragmentedPartTmpFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartTmpFile) DefragmentationFileUtils.defragmentedIndexFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexFile) IoStatisticsHolderNoOp(org.apache.ignite.internal.metric.IoStatisticsHolderNoOp) DataPageEvictionMode(org.apache.ignite.configuration.DataPageEvictionMode) DefragmentationFileUtils.batchRenameDefragmentedCacheGroupPartitions(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.batchRenameDefragmentedCacheGroupPartitions) FileVersionCheckingFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory) File(java.io.File) LongConsumer(java.util.function.LongConsumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) TreeMap(java.util.TreeMap) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) IntRWHashMap(org.apache.ignite.internal.util.collection.IntRWHashMap) IntHashMap(org.apache.ignite.internal.util.collection.IntHashMap) HashMap(java.util.HashMap) GridCacheOffheapManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager) PageStore(org.apache.ignite.internal.pagemem.store.PageStore) GridCompoundFuture(org.apache.ignite.internal.util.future.GridCompoundFuture) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) FileVersionCheckingFactory(org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory) IgniteException(org.apache.ignite.IgniteException) AbstractFreeList(org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList) List(java.util.List) DataPageEvictionMode(org.apache.ignite.configuration.DataPageEvictionMode) GridAtomicLong(org.apache.ignite.internal.util.GridAtomicLong) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IntHashMap(org.apache.ignite.internal.util.collection.IntHashMap) GridAtomicLong(org.apache.ignite.internal.util.GridAtomicLong) AtomicLong(java.util.concurrent.atomic.AtomicLong) CacheDataStore(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore) GridCacheDataStore(org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.GridCacheDataStore) IgniteCacheOffheapManager(org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) DefragmentationFileUtils.defragmentedPartMappingFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartMappingFile) DefragmentationFileUtils.defragmentedPartFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartFile) DefragmentationFileUtils.defragmentedIndexTmpFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexTmpFile) DefragmentationFileUtils.renameTempIndexFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.renameTempIndexFile) DefragmentationFileUtils.renameTempPartitionFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.renameTempPartitionFile) DefragmentationFileUtils.defragmentedPartTmpFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartTmpFile) DefragmentationFileUtils.defragmentedIndexFile(org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexFile) File(java.io.File)

Aggregations

File (java.io.File)1 Path (java.nio.file.Path)1 Arrays (java.util.Arrays)1 Comparator.comparing (java.util.Comparator.comparing)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 TreeMap (java.util.TreeMap)1 TreeSet (java.util.TreeSet)1 LinkedBlockingQueue (java.util.concurrent.LinkedBlockingQueue)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 LongConsumer (java.util.function.LongConsumer)1 Collectors (java.util.stream.Collectors)1 StreamSupport.stream (java.util.stream.StreamSupport.stream)1 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)1 IgniteException (org.apache.ignite.IgniteException)1