Examples with GridCacheDatabaseSharedManager org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager used on opensource projects

Example 21 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteWalRecoveryTest method testApplyDeltaRecords.

/**
 * @throws Exception if failed.
 */
public void testApplyDeltaRecords() throws Exception {
    try {
        IgniteEx ignite0 = (IgniteEx) startGrid("node0");
        ignite0.active(true);
        IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
        for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
        GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
        GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
        db.waitForCheckpoint("test");
        db.enableCheckpoints(false).get();
        // Log something to know where to start.
        WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
        info("Replay marker: " + ptr);
        for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
        info("Done puts...");
        for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
        info("Done removes...");
        for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
        info("Done puts...");
        Map<FullPageId, byte[]> rolledPages = new HashMap<>();
        int pageSize = sharedCtx.database().pageSize();
        ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
        // Now check that deltas can be correctly applied.
        try (WALIterator it = sharedCtx.wal().replay(ptr)) {
            while (it.hasNext()) {
                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
                WALRecord rec = tup.get2();
                if (rec instanceof PageSnapshot) {
                    PageSnapshot page = (PageSnapshot) rec;
                    rolledPages.put(page.fullPageId(), page.pageData());
                } else if (rec instanceof PageDeltaRecord) {
                    PageDeltaRecord delta = (PageDeltaRecord) rec;
                    FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
                    byte[] pageData = rolledPages.get(fullId);
                    if (pageData == null) {
                        pageData = new byte[pageSize];
                        rolledPages.put(fullId, pageData);
                    }
                    assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
                    buf.order(ByteOrder.nativeOrder());
                    buf.position(0);
                    buf.put(pageData);
                    buf.position(0);
                    delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
                    buf.position(0);
                    buf.get(pageData);
                }
            }
        }
        info("Done apply...");
        PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
        for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
            FullPageId fullId = entry.getKey();
            ignite0.context().cache().context().database().checkpointReadLock();
            try {
                long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), true);
                try {
                    long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
                    try {
                        byte[] data = entry.getValue();
                        for (int i = 0; i < data.length; i++) {
                            if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
                                // Skip tracking pages.
                                continue;
                            assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
                        }
                    } finally {
                        pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
                    }
                } finally {
                    pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
                }
            } finally {
                ignite0.context().cache().context().database().checkpointReadUnlock();
            }
        }
        ignite0.close();
    } finally {
        stopAllGrids();
    }
}

Example 22 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteCheckpointDirtyPagesForLowLoadTest method testManyCachesAndNotManyPuts.

/**
 * @throws Exception if failed.
 */
public void testManyCachesAndNotManyPuts() throws Exception {
    try {
        IgniteEx ignite = startGrid(0);
        ignite.active(true);
        log.info("Saving initial data to caches");
        for (int g = 0; g < GROUPS; g++) {
            for (int c = 0; c < CACHES_IN_GRP; c++) {
                ignite.cache("dummyCache" + c + "." + g).putAll(new TreeMap<Long, Long>() {

                    {
                        for (int j = 0; j < PARTS; j++) {
                            // to fill each partition cache with at least 1 element
                            put((long) j, (long) j);
                        }
                    }
                });
            }
        }
        GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (ignite.context().cache().context().database());
        Collection<Integer> pageCntObserved = new ArrayList<>();
        boolean checkpointWithLowNumOfPagesFound = false;
        for (int i = 0; i < 20; i++) {
            Random random = new Random();
            // touch some entry
            int d = random.nextInt(PARTS) + PARTS;
            int cIdx = random.nextInt(CACHES_IN_GRP);
            int gIdx = random.nextInt(GROUPS);
            String fullname = "dummyCache" + cIdx + "." + gIdx;
            ignite.cache(fullname).put(d, d);
            if (log.isInfoEnabled())
                log.info("Put to cache [" + fullname + "] value " + d);
            long start = System.nanoTime();
            try {
                final int cpTimeout = 25000;
                db.wakeupForCheckpoint("").get(cpTimeout, TimeUnit.MILLISECONDS);
            } catch (IgniteFutureTimeoutCheckedException ignored) {
                long msPassed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
                log.error("Timeout during waiting for checkpoint to start:" + " [" + msPassed + "] but checkpoint is not running");
                continue;
            }
            final int timeout = 5000;
            int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db, timeout);
            if (currCpPages < 0) {
                log.error("Timeout during waiting for checkpoint counter to be updated");
                continue;
            }
            pageCntObserved.add(currCpPages);
            log.info("Current CP pages: " + currCpPages);
            if (currCpPages < PARTS * GROUPS) {
                // reasonable number of pages in CP
                checkpointWithLowNumOfPagesFound = true;
                break;
            }
        }
        stopGrid(0);
        assertTrue("All checkpoints mark too much pages: " + pageCntObserved, checkpointWithLowNumOfPagesFound);
    } finally {
        stopAllGrids();
    }
}

Example 23 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method recoveryNoPageLost.

/**
 * @param checkpoint Checkpoint enable flag.
 * @throws Exception If failed.
 */
private void recoveryNoPageLost(boolean checkpoint) throws Exception {
    try {
        pageSize = 1024;
        extraCcfg = new CacheConfiguration(CACHE2_NAME);
        extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
        List<Integer> pages = null;
        AtomicInteger cnt = new AtomicInteger();
        for (int iter = 0; iter < 5; iter++) {
            log.info("Start node: " + iter);
            Ignite ignite = startGrid(0);
            ignite.cluster().active(true);
            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
            if (!checkpoint)
                dbMgr.enableCheckpoints(false).get();
            if (pages != null) {
                List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
                assertEquals(pages, curPags);
            }
            IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
            for (int i = 0; i < 128; i++) cache.put(cnt.incrementAndGet(), new byte[256 + iter * 100]);
            pages = allocatedPages(ignite, CACHE2_NAME);
            ignite.close();
        }
    } finally {
        stopAllGrids();
    }
}

Example 24 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testCheckpointHistory.

/**
 * @throws Exception if failed.
 */
public void testCheckpointHistory() throws Exception {
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    try {
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        dbMgr.waitForCheckpoint("test");
        // This number depends on wal history size.
        int entries = WAL_HIST_SIZE * 2;
        IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
        for (int i = 0; i < entries; i++) {
            // Put to partition 0.
            cache.put(i * PARTS, i * PARTS);
            // Put to partition 1.
            cache.put(i * PARTS + 1, i * PARTS + 1);
            dbMgr.waitForCheckpoint("test");
        }
        GridCacheDatabaseSharedManager.CheckpointHistory hist = dbMgr.checkpointHistory();
        assertTrue(hist.checkpoints().size() <= WAL_HIST_SIZE);
        File cpDir = dbMgr.checkpointDirectory();
        File[] cpFiles = cpDir.listFiles();
        // starts & ends + node_start
        assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2 + 1);
    } finally {
        stopAllGrids();
    }
}

Example 25 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testWalRecoveryRemoves.

/**
 * @throws Exception if failed.
 */
public void testWalRecoveryRemoves() throws Exception {
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    try {
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
        int txCnt = 100;
        int keysPerTx = 10;
        for (int i = 0; i < txCnt; i++) {
            try (Transaction tx = ignite.transactions().txStart()) {
                for (int j = 0; j < keysPerTx; j++) {
                    int k = i * keysPerTx + j;
                    cache.put(k, new IndexedValue(k));
                }
                tx.commit();
            }
        }
        for (int i = 0; i < txCnt; i++) {
            for (int j = 0; j < keysPerTx; j++) {
                int k = i * keysPerTx + j;
                assertEquals(k, cache.get(k).value());
            }
        }
        dbMgr.waitForCheckpoint("test");
        dbMgr.enableCheckpoints(false).get();
        for (int i = 0; i < txCnt / 2; i++) {
            try (Transaction tx = ignite.transactions().txStart()) {
                for (int j = 0; j < keysPerTx; j++) {
                    int k = i * keysPerTx + j;
                    cache.remove(k);
                }
                tx.commit();
            }
        }
        stopGrid();
        ignite = startGrid();
        ignite.cluster().active(true);
        cache = ignite.cache(CACHE_NAME);
        for (int i = 0; i < txCnt; i++) {
            for (int j = 0; j < keysPerTx; j++) {
                int k = i * keysPerTx + j;
                QueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
                List<List<?>> vals = cur.getAll();
                if (i < txCnt / 2) {
                    assertNull(cache.get(k));
                    assertTrue(F.isEmpty(vals));
                } else {
                    assertEquals(k, cache.get(k).value());
                    assertEquals(1, vals.size());
                    assertEquals("string-" + k, vals.get(0).get(0));
                }
            }
        }
    } finally {
        stopAllGrids();
    }
}