Examples with FilePageStore org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore used on opensource projects

Example 11 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgniteIndexReader method checkParts.

/**
 * Checks partitions, comparing partition indexes (cache data tree) to indexes given in {@code aTreesInfo}.
 *
 * @param aTreesInfo Index trees info to compare cache data tree with.
 * @return Map of errors, bound to partition id.
 */
private Map<Integer, List<Throwable>> checkParts(Map<String, TreeTraversalInfo> aTreesInfo) {
    System.out.println();
    // Map partId -> errors.
    Map<Integer, List<Throwable>> res = new HashMap<>();
    Map<String, TreeTraversalInfo> treesInfo = new HashMap<>(aTreesInfo);
    treesInfo.remove(META_TREE_NAME);
    ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Checking partitions", partCnt);
    for (int i = 0; i < partCnt; i++) {
        progressPrinter.printProgress();
        FilePageStore partStore = partStores[i];
        if (partStore == null)
            continue;
        List<Throwable> errors = new LinkedList<>();
        final int partId = i;
        try {
            long partMetaId = partMetaPageId(i, FLAG_DATA);
            doWithBuffer((buf, addr) -> {
                readPage(partStore, partMetaId, buf);
                PagePartitionMetaIO partMetaIO = PageIO.getPageIO(addr);
                long cacheDataTreeRoot = partMetaIO.getTreeRoot(addr);
                TreeTraversalInfo cacheDataTreeInfo = horizontalTreeScan(cacheDataTreeRoot, "dataTree-" + partId, new ItemsListStorage());
                for (Object dataTreeItem : cacheDataTreeInfo.itemStorage) {
                    CacheAwareLink cacheAwareLink = (CacheAwareLink) dataTreeItem;
                    for (Map.Entry<String, TreeTraversalInfo> e : treesInfo.entrySet()) {
                        String name = e.getKey();
                        TreeTraversalInfo tree = e.getValue();
                        int cacheId = getCacheId(name);
                        if (cacheId != cacheAwareLink.cacheId)
                            // It's index for other cache, don't check.
                            continue;
                        // Tombstones are not indexed and shouldn't be tested.
                        if (!tree.itemStorage.contains(cacheAwareLink) && !cacheAwareLink.tombstone)
                            errors.add(new IgniteException(cacheDataTreeEntryMissingError(name, cacheAwareLink)));
                    }
                    if (errors.size() >= CHECK_PARTS_MAX_ERRORS_PER_PARTITION) {
                        errors.add(new IgniteException("Too many errors (" + CHECK_PARTS_MAX_ERRORS_PER_PARTITION + ") found for partId=" + partId + ", stopping analysis for this partition."));
                        break;
                    }
                }
                return null;
            });
        } catch (IgniteCheckedException e) {
            errors.add(new IgniteException("Partition check failed, partId=" + i, e));
        }
        if (!errors.isEmpty())
            res.put(partId, errors);
    }
    return res;
}

Example 12 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgniteIndexReader method horizontalTreeScan.

/**
 * Traverse single index tree by each level horizontally.
 *
 * @param rootPageId Root page id.
 * @param treeName Tree name.
 * @param itemStorage Items storage.
 * @return Tree traversal info.
 */
private TreeTraversalInfo horizontalTreeScan(long rootPageId, String treeName, ItemStorage itemStorage) {
    FilePageStore store = filePageStore(partId(rootPageId));
    Map<Long, List<Throwable>> errors = new HashMap<>();
    Map<Class, Long> ioStat = new HashMap<>();
    TreeTraverseContext treeCtx = new TreeTraverseContext(treeName, store, ioStat, errors, null, null, null);
    ByteBuffer buf = allocateBuffer(pageSize);
    try {
        long addr = bufferAddress(buf);
        readPage(store, rootPageId, buf);
        PageIO pageIO = PageIO.getPageIO(addr);
        if (!(pageIO instanceof BPlusMetaIO))
            throw new IgniteException("Root page is not meta, pageId=" + rootPageId);
        BPlusMetaIO metaIO = (BPlusMetaIO) pageIO;
        ioStat.compute(metaIO.getClass(), (k, v) -> v == null ? 1 : v + 1);
        int lvlsCnt = metaIO.getLevelsCount(addr);
        long[] firstPageIds = IntStream.range(0, lvlsCnt).mapToLong(i -> metaIO.getFirstPageId(addr, i)).toArray();
        for (int i = 0; i < lvlsCnt; i++) {
            long pageId = firstPageIds[i];
            while (pageId > 0) {
                try {
                    buf.rewind();
                    readPage(store, pageId, buf);
                    pageIO = PageIO.getPageIO(addr);
                    if (i == 0 && !(pageIO instanceof BPlusLeafIO))
                        throw new IgniteException("Not-leaf page found on leaf level, pageId=" + pageId + ", level=" + i);
                    if (!(pageIO instanceof BPlusIO))
                        throw new IgniteException("Not-BPlus page found, pageId=" + pageId + ", level=" + i);
                    ioStat.compute(pageIO.getClass(), (k, v) -> v == null ? 1 : v + 1);
                    if (pageIO instanceof BPlusLeafIO) {
                        PageIOProcessor ioProcessor = getIOProcessor(pageIO);
                        PageContent pageContent = ioProcessor.getContent(pageIO, addr, pageId, treeCtx);
                        pageContent.items.forEach(itemStorage::add);
                    }
                    pageId = ((BPlusIO) pageIO).getForward(addr);
                } catch (Throwable e) {
                    errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e);
                    pageId = 0;
                }
            }
        }
    } catch (Throwable e) {
        errors.computeIfAbsent(rootPageId, k -> new LinkedList<>()).add(e);
    } finally {
        freeBuffer(buf);
    }
    return new TreeTraversalInfo(ioStat, errors, null, rootPageId, itemStorage);
}

Example 13 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class IgniteClusterSnapshotCheckTest method testClusterSnapshotCheckPartitionCounters.

/**
 * @throws Exception If fails.
 */
@Test
public void testClusterSnapshotCheckPartitionCounters() throws Exception {
    IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 1)), CACHE_KEYS_RANGE);
    ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get();
    Path part0 = U.searchFileRecursively(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME).toPath(), getPartitionFileName(PART_ID));
    assertNotNull(part0);
    assertTrue(part0.toString(), part0.toFile().exists());
    try (FilePageStore pageStore = (FilePageStore) ((FilePageStoreManager) ignite.context().cache().context().pageStore()).getPageStoreFactory(CU.cacheId(dfltCacheCfg.getName()), false).createPageStore(getTypeByPartId(PART_ID), () -> part0, val -> {
    })) {
        ByteBuffer buff = ByteBuffer.allocateDirect(ignite.configuration().getDataStorageConfiguration().getPageSize()).order(ByteOrder.nativeOrder());
        buff.clear();
        pageStore.read(0, buff, false);
        PagePartitionMetaIO io = PageIO.getPageIO(buff);
        long pageAddr = GridUnsafe.bufferAddress(buff);
        io.setUpdateCounter(pageAddr, CACHE_KEYS_RANGE * 2);
        pageStore.beginRecover();
        buff.flip();
        pageStore.write(PageIO.getPageId(buff), buff, 0, true);
        pageStore.finishRecover();
    }
    IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get();
    StringBuilder b = new StringBuilder();
    res.print(b::append, true);
    assertTrue(F.isEmpty(res.exceptions()));
    assertContains(log, b.toString(), "The check procedure has failed, conflict partitions has been found: [counterConflicts=1, hashConflicts=0]");
}

Example 14 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class PagesPossibleCorruptionDiagnosticTest method filePageStore.

/**
 * @param ignite Ignite instance.
 * @param partId Partition id.
 * @return File page store for given partition id.
 * @throws IgniteCheckedException If failed.
 */
private FilePageStore filePageStore(IgniteEx ignite, int partId) throws IgniteCheckedException {
    final PdsFolderSettings folderSettings = ignite.context().pdsFolderResolver().resolveFolders();
    File storeWorkDir = new File(folderSettings.persistentStoreRootPath(), folderSettings.folderName());
    File cacheWorkDir = new File(storeWorkDir, CACHE_DIR_PREFIX + DEFAULT_CACHE_NAME);
    File partFile = new File(cacheWorkDir, format(PART_FILE_TEMPLATE, partId));
    return (FilePageStore) storeFactory.createPageStore(FLAG_DATA, partFile, a -> {
    });
}

Example 15 with FilePageStore

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore in project ignite by apache.

the class AbstractEncryptionTest method checkGroupKey.

/**
 * Ensures that all pages of page store have expected encryption key identifier.
 *
 * @param grpId Cache group ID.
 * @param expKeyId Encryption key ID.
 * @param timeout Timeout to wait for encryption to complete.
 * @throws Exception If failed.
 */
protected void checkGroupKey(int grpId, int expKeyId, long timeout) throws Exception {
    awaitEncryption(G.allGrids(), grpId, timeout);
    for (Ignite g : G.allGrids()) {
        IgniteEx grid = (IgniteEx) g;
        if (grid.context().clientNode())
            continue;
        info("Validating encryption key [node=" + g.cluster().localNode().id() + ", grp=" + grpId + "]");
        CacheGroupContext grp = grid.context().cache().cacheGroup(grpId);
        if (grp == null || !grp.affinityNode()) {
            info("Context doesn't exits on " + grid.localNode().id());
            continue;
        }
        GridEncryptionManager encryption = grid.context().encryption();
        assertEquals(grid.localNode().id().toString(), (byte) expKeyId, encryption.getActiveKey(grpId).id());
        IgniteInternalFuture<Void> fut = encryption.reencryptionFuture(grpId);
        // The future will be completed after the checkpoint, forcecheckpoint does nothing
        // if the checkpoint has already been scheduled.
        GridTestUtils.waitForCondition(() -> {
            if (fut.isDone())
                return true;
            try {
                forceCheckpoint(g);
            } catch (IgniteCheckedException e) {
                throw new RuntimeException(e);
            }
            return fut.isDone();
        }, timeout);
        assertTrue(fut.isDone());
        assertEquals(0, encryption.getBytesLeftForReencryption(grpId));
        List<Integer> parts = IntStream.range(0, grp.shared().affinity().affinity(grpId).partitions()).boxed().collect(Collectors.toList());
        parts.add(INDEX_PARTITION);
        int realPageSize = grp.dataRegion().pageMemory().realPageSize(grpId);
        int encryptionBlockSize = grp.shared().kernalContext().config().getEncryptionSpi().blockSize();
        for (int p : parts) {
            FilePageStore pageStore = (FilePageStore) ((FilePageStoreManager) grp.shared().pageStore()).getStore(grpId, p);
            if (!pageStore.exists())
                continue;
            long state = grid.context().encryption().getEncryptionState(grpId, p);
            String msg = String.format("p=%d, off=%d, total=%d", p, ReencryptStateUtils.pageIndex(state), ReencryptStateUtils.pageCount(state));
            assertEquals(msg, 0, ReencryptStateUtils.pageCount(state));
            assertEquals(msg, 0, ReencryptStateUtils.pageIndex(state));
            long startPageId = PageIdUtils.pageId(p, PageIdAllocator.FLAG_DATA, 0);
            int pagesCnt = pageStore.pages();
            int pageSize = pageStore.getPageSize();
            ByteBuffer pageBuf = ByteBuffer.allocate(pageSize);
            Path path = new File(pageStore.getFileAbsolutePath()).toPath();
            try (FileChannel ch = FileChannel.open(path, StandardOpenOption.READ)) {
                for (int n = 0; n < pagesCnt; n++) {
                    long pageId = startPageId + n;
                    long pageOff = pageStore.pageOffset(pageId);
                    pageBuf.position(0);
                    ch.position(pageOff);
                    ch.read(pageBuf);
                    pageBuf.position(realPageSize + encryptionBlockSize);
                    int pageCrc = pageBuf.getInt();
                    int pageKeyId = pageBuf.get() & 0xff;
                    // If this page is empty we can skip it.
                    if (pageCrc == 0 && pageKeyId == 0) {
                        pageBuf.position(0);
                        boolean emptyPage = false;
                        while (pageBuf.hasRemaining() && !emptyPage) emptyPage = pageBuf.getLong() == 0;
                        if (emptyPage)
                            continue;
                    }
                    msg = String.format("File=%s, page=%d", pageStore.getFileAbsolutePath(), n);
                    assertEquals(msg, expKeyId, pageKeyId);
                }
            }
        }
    }
}