Examples with FilePageStore org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore used on opensource projects

Example 1 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IdleVerifyUtility method checkPartitionsPageCrcSum.

/**
 * Checks CRC sum of pages with {@code pageType} page type stored in partition with {@code partId} id
 * and associated with cache group.
 *
 * @param pageStoreSup Page store supplier.
 * @param partId Partition id.
 * @param pageType Page type. Possible types {@link PageIdAllocator#FLAG_DATA}, {@link PageIdAllocator#FLAG_IDX}
 *      and {@link PageIdAllocator#FLAG_AUX}.
 */
public static void checkPartitionsPageCrcSum(IgniteThrowableSupplier<FilePageStore> pageStoreSup, int partId, byte pageType) {
    assert pageType == FLAG_DATA || pageType == FLAG_IDX || pageType == FLAG_AUX : pageType;
    FilePageStore pageStore = null;
    try {
        pageStore = pageStoreSup.get();
        long pageId = PageIdUtils.pageId(partId, (byte) 0, 0);
        ByteBuffer buf = ByteBuffer.allocateDirect(pageStore.getPageSize()).order(ByteOrder.nativeOrder());
        for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) {
            buf.clear();
            pageStore.read(pageId, buf, true, true);
        }
    } catch (Throwable e) {
        String msg0 = "CRC check of partition failed [partId=" + partId + ", grpName=" + (pageStore == null ? "" : cacheGroupName(new File(pageStore.getFileAbsolutePath()).getParentFile())) + ", part=" + (pageStore == null ? "" : pageStore.getFileAbsolutePath()) + ']';
        throw new IgniteException(msg0, e);
    }
}

Example 2 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgniteIndexReader method traverseTree.

/**
 * Traverse single index tree from root to leafs.
 *
 * @param rootPageId Root page id.
 * @param treeName Tree name.
 * @param innerCb Inner pages callback.
 * @param leafCb Leaf pages callback.
 * @param itemCb Items callback.
 * @param itemStorage Items storage.
 * @return Tree traversal info.
 */
TreeTraversalInfo traverseTree(long rootPageId, String treeName, @Nullable PageCallback innerCb, @Nullable PageCallback leafCb, @Nullable ItemCallback itemCb, ItemStorage itemStorage) {
    FilePageStore store = filePageStore(partId(rootPageId));
    Map<Class, Long> ioStat = new HashMap<>();
    Map<Long, List<Throwable>> errors = new HashMap<>();
    Set<Long> innerPageIds = new HashSet<>();
    PageCallback innerCb0 = (content, pageId) -> {
        if (innerCb != null)
            innerCb.cb(content, pageId);
        innerPageIds.add(normalizePageId(pageId));
    };
    ItemCallback itemCb0 = (currPageId, item, link) -> {
        if (itemCb != null)
            itemCb.cb(currPageId, item, link);
        itemStorage.add(item);
    };
    getTreeNode(rootPageId, new TreeTraverseContext(treeName, store, ioStat, errors, innerCb0, leafCb, itemCb0));
    return new TreeTraversalInfo(ioStat, errors, innerPageIds, rootPageId, itemStorage);
}

Example 3 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgniteIndexReaderFilePageStoreFactoryImpl method headerBuffer.

/**
 * {@inheritDoc}
 */
@Override
public ByteBuffer headerBuffer(byte type) throws IgniteCheckedException {
    int ver = storeFactory.latestVersion();
    FilePageStore store = (FilePageStore) storeFactory.createPageStore(type, (IgniteOutClosure<Path>) null, allocationTracker::add);
    return store.header(type, storeFactory.headerSize(ver));
}

Example 4 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class SnapshotPartitionsVerifyHandler method invoke.

/**
 * {@inheritDoc}
 */
@Override
public Map<PartitionKeyV2, PartitionHashRecordV2> invoke(SnapshotHandlerContext opCtx) throws IgniteCheckedException {
    SnapshotMetadata meta = opCtx.metadata();
    Set<Integer> grps = F.isEmpty(opCtx.groups()) ? new HashSet<>(meta.partitions().keySet()) : opCtx.groups().stream().map(CU::cacheId).collect(Collectors.toSet());
    Set<File> partFiles = new HashSet<>();
    IgniteSnapshotManager snpMgr = cctx.snapshotMgr();
    for (File dir : snpMgr.snapshotCacheDirectories(meta.snapshotName(), meta.folderName())) {
        int grpId = CU.cacheId(cacheGroupName(dir));
        if (!grps.remove(grpId))
            continue;
        Set<Integer> parts = meta.partitions().get(grpId) == null ? Collections.emptySet() : new HashSet<>(meta.partitions().get(grpId));
        for (File part : cachePartitionFiles(dir)) {
            int partId = partId(part.getName());
            if (!parts.remove(partId))
                continue;
            partFiles.add(part);
        }
        if (!parts.isEmpty()) {
            throw new IgniteException("Snapshot data doesn't contain required cache group partition " + "[grpId=" + grpId + ", snpName=" + meta.snapshotName() + ", consId=" + meta.consistentId() + ", missed=" + parts + ", meta=" + meta + ']');
        }
    }
    if (!grps.isEmpty()) {
        throw new IgniteException("Snapshot data doesn't contain required cache groups " + "[grps=" + grps + ", snpName=" + meta.snapshotName() + ", consId=" + meta.consistentId() + ", meta=" + meta + ']');
    }
    Map<PartitionKeyV2, PartitionHashRecordV2> res = new ConcurrentHashMap<>();
    ThreadLocal<ByteBuffer> buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()).order(ByteOrder.nativeOrder()));
    GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(meta.snapshotName(), meta.folderName());
    for (GridComponent comp : snpCtx) comp.start();
    try {
        U.doInParallel(snpMgr.snapshotExecutorService(), partFiles, part -> {
            String grpName = cacheGroupName(part.getParentFile());
            int grpId = CU.cacheId(grpName);
            int partId = partId(part.getName());
            FilePageStoreManager storeMgr = (FilePageStoreManager) cctx.pageStore();
            try (FilePageStore pageStore = (FilePageStore) storeMgr.getPageStoreFactory(grpId, false).createPageStore(getTypeByPartId(partId), part::toPath, val -> {
            })) {
                if (partId == INDEX_PARTITION) {
                    checkPartitionsPageCrcSum(() -> pageStore, INDEX_PARTITION, FLAG_IDX);
                    return null;
                }
                if (grpId == MetaStorage.METASTORAGE_CACHE_ID) {
                    checkPartitionsPageCrcSum(() -> pageStore, partId, FLAG_DATA);
                    return null;
                }
                ByteBuffer pageBuff = buff.get();
                pageBuff.clear();
                pageStore.read(0, pageBuff, true);
                long pageAddr = GridUnsafe.bufferAddress(pageBuff);
                PagePartitionMetaIO io = PageIO.getPageIO(pageBuff);
                GridDhtPartitionState partState = fromOrdinal(io.getPartitionState(pageAddr));
                if (partState != OWNING) {
                    throw new IgniteCheckedException("Snapshot partitions must be in the OWNING " + "state only: " + partState);
                }
                long updateCntr = io.getUpdateCounter(pageAddr);
                long size = io.getSize(pageAddr);
                if (log.isDebugEnabled()) {
                    log.debug("Partition [grpId=" + grpId + ", id=" + partId + ", counter=" + updateCntr + ", size=" + size + "]");
                }
                // Snapshot partitions must always be in OWNING state.
                // There is no `primary` partitions for snapshot.
                PartitionKeyV2 key = new PartitionKeyV2(grpId, partId, grpName);
                PartitionHashRecordV2 hash = calculatePartitionHash(key, updateCntr, meta.consistentId(), GridDhtPartitionState.OWNING, false, size, snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore));
                assert hash != null : "OWNING must have hash: " + key;
                res.put(key, hash);
            } catch (IOException e) {
                throw new IgniteCheckedException(e);
            }
            return null;
        });
    } catch (Throwable t) {
        log.error("Error executing handler: ", t);
        throw t;
    } finally {
        for (GridComponent comp : snpCtx) comp.stop(true);
    }
    return res;
}

Example 5 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgnitePdsDataRegionMetricsTest method checkMetricsConsistency.

/**
 */
private void checkMetricsConsistency(final IgniteEx node, String cacheName) throws Exception {
    FilePageStoreManager pageStoreMgr = (FilePageStoreManager) node.context().cache().context().pageStore();
    assert pageStoreMgr != null : "Persistence is not enabled";
    boolean metaStore = METASTORAGE_CACHE_NAME.equals(cacheName);
    boolean txLog = TX_LOG_CACHE_NAME.equals(cacheName);
    File cacheWorkDir = metaStore ? new File(pageStoreMgr.workDir(), METASTORAGE_DIR_NAME) : txLog ? new File(pageStoreMgr.workDir(), TX_LOG_CACHE_NAME) : pageStoreMgr.cacheWorkDir(node.cachex(cacheName).configuration());
    long totalPersistenceSize = 0;
    try (DirectoryStream<Path> files = newDirectoryStream(cacheWorkDir.toPath(), entry -> entry.toFile().getName().endsWith(".bin"))) {
        for (Path path : files) {
            File file = path.toFile();
            FilePageStore store = (FilePageStore) pageStoreMgr.getStore(metaStore ? METASTORAGE_CACHE_ID : CU.cacheId(cacheName), partId(file.getName()));
            int pageSize = store.getPageSize();
            long storeSize = path.toFile().length() - store.headerSize();
            if (storeSize % pageSize != 0)
                // Adjust for possible page compression.
                storeSize = (storeSize / pageSize + 1) * pageSize;
            totalPersistenceSize += storeSize;
        }
    }
    GridCacheSharedContext cctx = node.context().cache().context();
    String regionName = metaStore ? GridCacheDatabaseSharedManager.METASTORE_DATA_REGION_NAME : txLog ? TX_LOG_CACHE_NAME : cctx.cacheContext(CU.cacheId(cacheName)).group().dataRegion().config().getName();
    long totalAllocatedPagesFromMetrics = cctx.database().memoryMetrics(regionName).getTotalAllocatedPages();
    assertEquals("Number of allocated pages is different than in metrics for [node=" + node.name() + ", cache=" + cacheName + "]", totalPersistenceSize / pageStoreMgr.pageSize(), totalAllocatedPagesFromMetrics);
}