Examples with PageStore org.apache.ignite.internal.pagemem.store.PageStore used on opensource projects

Example 21 with PageStore

use of org.apache.ignite.internal.pagemem.store.PageStore in project ignite by apache.

the class FilePageStoreManager method truncate.

/**
 * {@inheritDoc}
 */
@Override
public void truncate(int grpId, int partId, int tag) throws IgniteCheckedException {
    assert partId <= MAX_PARTITION_ID;
    PageStore store = getStore(grpId, partId);
    store.truncate(tag);
}

Example 22 with PageStore

use of org.apache.ignite.internal.pagemem.store.PageStore in project ignite by apache.

the class FilePageStoreManager method initDir.

/**
 * @param cacheWorkDir Work directory.
 * @param grpId Group ID.
 * @param partitions Number of partitions.
 * @param pageMetrics Page metrics.
 * @param encrypted {@code True} if this cache encrypted.
 * @return Cache store holder.
 * @throws IgniteCheckedException If failed.
 */
private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions, PageMetrics pageMetrics, boolean encrypted) throws IgniteCheckedException {
    try {
        boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
        if (dirExisted) {
            MaintenanceRegistry mntcReg = cctx.kernalContext().maintenanceRegistry();
            if (!mntcReg.isMaintenanceMode())
                DefragmentationFileUtils.beforeInitPageStores(cacheWorkDir, log);
        }
        File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
        if (dirExisted && !idxFile.exists())
            grpsWithoutIdx.add(grpId);
        FileVersionCheckingFactory pageStoreFactory = getPageStoreFactory(grpId, encrypted);
        PageStore idxStore = pageStoreFactory.createPageStore(PageStore.TYPE_IDX, idxFile, pageMetrics.totalPages()::add);
        PageStore[] partStores = new PageStore[partitions];
        for (int partId = 0; partId < partStores.length; partId++) {
            final int p = partId;
            PageStore partStore = pageStoreFactory.createPageStore(PageStore.TYPE_DATA, () -> getPartitionFilePath(cacheWorkDir, p), pageMetrics.totalPages()::add);
            partStores[partId] = partStore;
        }
        return new CacheStoreHolder(idxStore, partStores);
    } catch (IgniteCheckedException e) {
        if (X.hasCause(e, StorageException.class, IOException.class))
            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
        throw e;
    }
}

Example 23 with PageStore

use of org.apache.ignite.internal.pagemem.store.PageStore in project ignite by apache.

the class FilePageStoreManager method beginRecover.

/**
 * {@inheritDoc}
 */
@Override
public void beginRecover() {
    List<String> groupsWithWalDisabled = checkCachesWithDisabledWal();
    if (!groupsWithWalDisabled.isEmpty()) {
        String errorMsg = "Cache groups with potentially corrupted partition files found. " + "To cleanup them maintenance is needed, node will enter maintenance mode on next restart. " + "Cleanup cache group folders manually or trigger maintenance action to do that and restart the node. " + "Corrupted files are located in subdirectories " + groupsWithWalDisabled + " in a work dir " + storeWorkDir;
        log.warning(errorMsg);
        try {
            cctx.kernalContext().maintenanceRegistry().registerMaintenanceTask(new MaintenanceTask(CORRUPTED_DATA_FILES_MNTC_TASK_NAME, "Corrupted cache groups found", groupsWithWalDisabled.stream().collect(Collectors.joining(File.separator))));
        } catch (IgniteCheckedException e) {
            log.warning("Failed to register maintenance record for corrupted partition files.", e);
        }
        throw new IgniteException(errorMsg);
    }
    for (CacheStoreHolder holder : idxCacheStores.values()) {
        holder.idxStore.beginRecover();
        for (PageStore partStore : holder.partStores) partStore.beginRecover();
    }
}

Example 24 with PageStore

use of org.apache.ignite.internal.pagemem.store.PageStore in project ignite by apache.

the class IgniteSnapshotManager method partitionRowIterator.

/**
 * @param snpName Snapshot name.
 * @param folderName The node folder name, usually it's the same as the U.maskForFileName(consistentId).
 * @param grpName Cache group name.
 * @param partId Partition id.
 * @return Iterator over partition.
 * @throws IgniteCheckedException If and error occurs.
 */
public GridCloseableIterator<CacheDataRow> partitionRowIterator(GridKernalContext ctx, String snpName, String folderName, String grpName, int partId) throws IgniteCheckedException {
    File snpDir = resolveSnapshotDir(snpName);
    File nodePath = new File(snpDir, databaseRelativePath(folderName));
    if (!nodePath.exists())
        throw new IgniteCheckedException("Consistent id directory doesn't exists: " + nodePath.getAbsolutePath());
    List<File> grps = cacheDirectories(nodePath, name -> name.equals(grpName));
    if (F.isEmpty(grps)) {
        throw new IgniteCheckedException("The snapshot cache group not found [dir=" + snpDir.getAbsolutePath() + ", grpName=" + grpName + ']');
    }
    if (grps.size() > 1) {
        throw new IgniteCheckedException("The snapshot cache group directory cannot be uniquely identified [dir=" + snpDir.getAbsolutePath() + ", grpName=" + grpName + ']');
    }
    File snpPart = getPartitionFile(new File(snapshotLocalDir(snpName), databaseRelativePath(folderName)), grps.get(0).getName(), partId);
    int grpId = CU.cacheId(grpName);
    FilePageStore pageStore = (FilePageStore) storeMgr.getPageStoreFactory(grpId, cctx.cache().isEncrypted(grpId)).createPageStore(getTypeByPartId(partId), snpPart::toPath, val -> {
    });
    GridCloseableIterator<CacheDataRow> partIter = partitionRowIterator(ctx, grpName, partId, pageStore);
    return new GridCloseableIteratorAdapter<CacheDataRow>() {

        /**
         * {@inheritDoc}
         */
        @Override
        protected CacheDataRow onNext() throws IgniteCheckedException {
            return partIter.nextX();
        }

        /**
         * {@inheritDoc}
         */
        @Override
        protected boolean onHasNext() throws IgniteCheckedException {
            return partIter.hasNextX();
        }

        /**
         * {@inheritDoc}
         */
        @Override
        protected void onClose() {
            U.closeQuiet(pageStore);
        }
    };
}

Example 25 with PageStore

use of org.apache.ignite.internal.pagemem.store.PageStore in project ignite by apache.

the class CachePartitionDefragmentationManager method executeDefragmentation.

/**
 */
public void executeDefragmentation() throws IgniteCheckedException {
    Map<Integer, List<CacheDataStore>> oldStores = new HashMap<>();
    for (CacheGroupContext oldGrpCtx : cacheGrpCtxsForDefragmentation) {
        int grpId = oldGrpCtx.groupId();
        final IgniteCacheOffheapManager offheap = oldGrpCtx.offheap();
        List<CacheDataStore> oldCacheDataStores = stream(offheap.cacheDataStores().spliterator(), false).filter(store -> {
            try {
                return filePageStoreMgr.exists(grpId, store.partId());
            } catch (IgniteCheckedException e) {
                throw new IgniteException(e);
            }
        }).collect(Collectors.toList());
        oldStores.put(grpId, oldCacheDataStores);
    }
    int partitionCount = oldStores.values().stream().mapToInt(List::size).sum();
    status.onStart(cacheGrpCtxsForDefragmentation, partitionCount);
    try {
        // Now the actual process starts.
        IgniteInternalFuture<?> idxDfrgFut = null;
        DataPageEvictionMode prevPageEvictionMode = null;
        for (CacheGroupContext oldGrpCtx : cacheGrpCtxsForDefragmentation) {
            int grpId = oldGrpCtx.groupId();
            File workDir = filePageStoreMgr.cacheWorkDir(oldGrpCtx.sharedGroup(), oldGrpCtx.cacheOrGroupName());
            List<CacheDataStore> oldCacheDataStores = oldStores.get(grpId);
            if (skipAlreadyDefragmentedCacheGroup(workDir, grpId, log)) {
                status.onCacheGroupSkipped(oldGrpCtx, oldCacheDataStores.size());
                continue;
            }
            try {
                GridCacheOffheapManager offheap = (GridCacheOffheapManager) oldGrpCtx.offheap();
                status.onCacheGroupStart(oldGrpCtx, oldCacheDataStores.size());
                if (workDir == null || oldCacheDataStores.isEmpty()) {
                    status.onCacheGroupFinish(oldGrpCtx);
                    continue;
                }
                // We can't start defragmentation of new group on the region that has wrong eviction mode.
                // So waiting of the previous cache group defragmentation is inevitable.
                DataPageEvictionMode curPageEvictionMode = oldGrpCtx.dataRegion().config().getPageEvictionMode();
                if (prevPageEvictionMode == null || prevPageEvictionMode != curPageEvictionMode) {
                    prevPageEvictionMode = curPageEvictionMode;
                    partDataRegion.config().setPageEvictionMode(curPageEvictionMode);
                    if (idxDfrgFut != null)
                        idxDfrgFut.get();
                }
                IntMap<CacheDataStore> cacheDataStores = new IntHashMap<>();
                for (CacheDataStore store : offheap.cacheDataStores()) {
                    // This would mean that these partitions are empty.
                    assert store.tree() == null || store.tree().groupId() == grpId;
                    if (store.tree() != null)
                        cacheDataStores.put(store.partId(), store);
                }
                dbMgr.checkpointedDataRegions().remove(oldGrpCtx.dataRegion());
                // Another cheat. Ttl cleanup manager knows too much shit.
                oldGrpCtx.caches().stream().filter(cacheCtx -> cacheCtx.groupId() == grpId).forEach(cacheCtx -> cacheCtx.ttl().unregister());
                // Technically wal is already disabled, but "PageHandler.isWalDeltaRecordNeeded" doesn't care
                // and WAL records will be allocated anyway just to be ignored later if we don't disable WAL for
                // cache group explicitly.
                oldGrpCtx.localWalEnabled(false, false);
                boolean encrypted = oldGrpCtx.config().isEncryptionEnabled();
                FileVersionCheckingFactory pageStoreFactory = filePageStoreMgr.getPageStoreFactory(grpId, encrypted);
                AtomicLong idxAllocationTracker = new GridAtomicLong();
                createIndexPageStore(grpId, workDir, pageStoreFactory, partDataRegion, idxAllocationTracker::addAndGet);
                checkCancellation();
                GridCompoundFuture<Object, Object> cmpFut = new GridCompoundFuture<>();
                PageMemoryEx oldPageMem = (PageMemoryEx) oldGrpCtx.dataRegion().pageMemory();
                CacheGroupContext newGrpCtx = new CacheGroupContext(sharedCtx, grpId, oldGrpCtx.receivedFrom(), CacheType.USER, oldGrpCtx.config(), oldGrpCtx.affinityNode(), partDataRegion, oldGrpCtx.cacheObjectContext(), null, null, oldGrpCtx.localStartVersion(), true, false, true);
                defragmentationCheckpoint.checkpointTimeoutLock().checkpointReadLock();
                try {
                    // This will initialize partition meta in index partition - meta tree and reuse list.
                    newGrpCtx.start();
                } finally {
                    defragmentationCheckpoint.checkpointTimeoutLock().checkpointReadUnlock();
                }
                IgniteUtils.doInParallel(defragmentationThreadPool, oldCacheDataStores, oldCacheDataStore -> defragmentOnePartition(oldGrpCtx, grpId, workDir, offheap, pageStoreFactory, cmpFut, oldPageMem, newGrpCtx, oldCacheDataStore));
                // A bit too general for now, but I like it more then saving only the last checkpoint future.
                cmpFut.markInitialized().get();
                idxDfrgFut = new GridFinishedFuture<>();
                if (filePageStoreMgr.hasIndexStore(grpId)) {
                    defragmentIndexPartition(oldGrpCtx, newGrpCtx);
                    idxDfrgFut = defragmentationCheckpoint.forceCheckpoint("index defragmented", null).futureFor(FINISHED);
                }
                PageStore oldIdxPageStore = filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
                idxDfrgFut = idxDfrgFut.chain(fut -> {
                    if (log.isDebugEnabled()) {
                        log.debug(S.toString("Index partition defragmented", "grpId", grpId, false, "oldPages", oldIdxPageStore.pages(), false, "newPages", idxAllocationTracker.get() + 1, false, "pageSize", pageSize, false, "partFile", defragmentedIndexFile(workDir).getName(), false, "workDir", workDir, false));
                    }
                    oldPageMem.invalidate(grpId, INDEX_PARTITION);
                    PageMemoryEx partPageMem = (PageMemoryEx) partDataRegion.pageMemory();
                    partPageMem.invalidate(grpId, INDEX_PARTITION);
                    DefragmentationPageReadWriteManager pageMgr = (DefragmentationPageReadWriteManager) partPageMem.pageManager();
                    pageMgr.pageStoreMap().removePageStore(grpId, INDEX_PARTITION);
                    PageMemoryEx mappingPageMem = (PageMemoryEx) mappingDataRegion.pageMemory();
                    pageMgr = (DefragmentationPageReadWriteManager) mappingPageMem.pageManager();
                    pageMgr.pageStoreMap().clear(grpId);
                    renameTempIndexFile(workDir);
                    writeDefragmentationCompletionMarker(filePageStoreMgr.getPageStoreFileIoFactory(), workDir, log);
                    batchRenameDefragmentedCacheGroupPartitions(workDir, log);
                    return null;
                });
                status.onIndexDefragmented(oldGrpCtx, oldIdxPageStore.size(), // + file header.
                pageSize + idxAllocationTracker.get() * pageSize);
            } catch (DefragmentationCancelledException e) {
                DefragmentationFileUtils.deleteLeftovers(workDir);
                throw e;
            }
            status.onCacheGroupFinish(oldGrpCtx);
        }
        if (idxDfrgFut != null)
            idxDfrgFut.get();
        mntcReg.unregisterMaintenanceTask(DEFRAGMENTATION_MNTC_TASK_NAME);
        status.onFinish();
        completionFut.onDone();
    } catch (DefragmentationCancelledException e) {
        mntcReg.unregisterMaintenanceTask(DEFRAGMENTATION_MNTC_TASK_NAME);
        log.info("Defragmentation process has been cancelled.");
        status.onFinish();
        completionFut.onDone();
    } catch (Throwable t) {
        log.error("Defragmentation process failed.", t);
        status.onFinish();
        completionFut.onDone(t);
        throw t;
    } finally {
        defragmentationCheckpoint.stop(true);
    }
}