Examples with GridCacheDatabaseSharedManager org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager used on opensource projects

Example 6 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgnitePdsNoActualWalHistoryTest method testWalBig.

/**
 * @throws Exception if failed.
 */
public void testWalBig() throws Exception {
    try {
        IgniteEx ignite = startGrid(1);
        ignite.active(true);
        IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
        Random rnd = new Random();
        Map<Integer, IndexedObject> map = new HashMap<>();
        for (int i = 0; i < 40_000; i++) {
            if (i % 1000 == 0)
                X.println(" >> " + i);
            int k = rnd.nextInt(300_000);
            IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
            cache.put(k, v);
            map.put(k, v);
        }
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ignite.context().cache().context().database();
        // Create many checkpoints to clean up the history.
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.wakeupForCheckpoint("test").get();
        dbMgr.enableCheckpoints(false).get();
        for (int i = 0; i < 50; i++) {
            int k = rnd.nextInt(300_000);
            IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
            cache.put(k, v);
            map.put(k, v);
        }
        stopGrid(1);
        ignite = startGrid(1);
        ignite.active(true);
        cache = ignite.cache(CACHE_NAME);
        // Check.
        for (Integer k : map.keySet()) assertEquals(map.get(k), cache.get(k));
    } finally {
        stopAllGrids();
    }
}

Example 7 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class MockWalIteratorFactory method iterator.

/**
 * Creates iterator
 * @param wal WAL directory without node consistent id
 * @param walArchive WAL archive without node consistent id
 * @return iterator
 * @throws IgniteCheckedException if IO failed
 */
@SuppressWarnings("unchecked")
public WALIterator iterator(File wal, File walArchive) throws IgniteCheckedException {
    final DataStorageConfiguration persistentCfg1 = Mockito.mock(DataStorageConfiguration.class);
    when(persistentCfg1.getWalPath()).thenReturn(wal.getAbsolutePath());
    when(persistentCfg1.getWalArchivePath()).thenReturn(walArchive.getAbsolutePath());
    when(persistentCfg1.getWalSegments()).thenReturn(segments);
    when(persistentCfg1.getWalBufferSize()).thenReturn(DataStorageConfiguration.DFLT_WAL_BUFF_SIZE);
    when(persistentCfg1.getWalRecordIteratorBufferSize()).thenReturn(DataStorageConfiguration.DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE);
    final FileIOFactory fileIOFactory = new DataStorageConfiguration().getFileIOFactory();
    when(persistentCfg1.getFileIOFactory()).thenReturn(fileIOFactory);
    final IgniteConfiguration cfg = Mockito.mock(IgniteConfiguration.class);
    when(cfg.getDataStorageConfiguration()).thenReturn(persistentCfg1);
    final GridKernalContext ctx = Mockito.mock(GridKernalContext.class);
    when(ctx.config()).thenReturn(cfg);
    when(ctx.clientNode()).thenReturn(false);
    when(ctx.pdsFolderResolver()).thenReturn(new PdsFoldersResolver() {

        @Override
        public PdsFolderSettings resolveFolders() {
            return new PdsFolderSettings(new File("."), subfolderName, consistentId, null, false);
        }
    });
    final GridDiscoveryManager disco = Mockito.mock(GridDiscoveryManager.class);
    when(ctx.discovery()).thenReturn(disco);
    final IgniteWriteAheadLogManager mgr = new FileWriteAheadLogManager(ctx);
    final GridCacheSharedContext sctx = Mockito.mock(GridCacheSharedContext.class);
    when(sctx.kernalContext()).thenReturn(ctx);
    when(sctx.discovery()).thenReturn(disco);
    when(sctx.gridConfig()).thenReturn(cfg);
    final GridCacheDatabaseSharedManager db = Mockito.mock(GridCacheDatabaseSharedManager.class);
    when(db.pageSize()).thenReturn(pageSize);
    when(sctx.database()).thenReturn(db);
    when(sctx.logger(any(Class.class))).thenReturn(log);
    mgr.start(sctx);
    return mgr.replay(null);
}

Example 8 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testHistoricalRebalanceIterator.

/**
 * @throws Exception if failed.
 */
public void testHistoricalRebalanceIterator() throws Exception {
    System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
    extraCcfg = new CacheConfiguration(CACHE_NAME + "2");
    extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
    Ignite ignite = startGrid();
    try {
        ignite.cluster().active(true);
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        dbMgr.waitForCheckpoint("test");
        // This number depends on wal history size.
        int entries = 25;
        IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
        IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE_NAME + "2");
        for (int i = 0; i < entries; i++) {
            // Put to partition 0.
            cache.put(i * PARTS, i * PARTS);
            // Put to partition 1.
            cache.put(i * PARTS + 1, i * PARTS + 1);
            // Put to another cache.
            cache2.put(i, i);
            dbMgr.waitForCheckpoint("test");
        }
        for (int i = 0; i < entries; i++) {
            assertEquals((Integer) (i * PARTS), cache.get(i * PARTS));
            assertEquals((Integer) (i * PARTS + 1), cache.get(i * PARTS + 1));
            assertEquals((Integer) (i), cache2.get(i));
        }
        CacheGroupContext grp = ((IgniteEx) ignite).context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
        IgniteCacheOffheapManager offh = grp.offheap();
        AffinityTopologyVersion topVer = grp.affinity().lastVersion();
        IgniteDhtDemandedPartitionsMap map;
        for (int i = 0; i < entries; i++) {
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(0, i, Long.MAX_VALUE, entries);
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(0));
                for (int j = i; j < entries; j++) {
                    assertTrue("i=" + i + ", j=" + j, it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            }
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(1, i, Long.MAX_VALUE, entries);
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(1));
                for (int j = i; j < entries; j++) {
                    assertTrue(it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS + 1, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS + 1, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            }
        }
        stopAllGrids();
        // Check that iterator is valid after restart.
        ignite = startGrid();
        ignite.cluster().active(true);
        grp = ((IgniteEx) ignite).context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
        offh = grp.offheap();
        topVer = grp.affinity().lastVersion();
        for (int i = 0; i < entries; i++) {
            long start = System.currentTimeMillis();
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(0, i, Long.MAX_VALUE, entries);
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                long end = System.currentTimeMillis();
                info("Time to get iterator: " + (end - start));
                assertTrue("Not historical for iteration: " + i, it.historical(0));
                assertNotNull(it);
                start = System.currentTimeMillis();
                for (int j = i; j < entries; j++) {
                    assertTrue("i=" + i + ", j=" + j, it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                end = System.currentTimeMillis();
                info("Time to iterate: " + (end - start));
                assertFalse(it.hasNext());
            }
            map = new IgniteDhtDemandedPartitionsMap();
            map.addHistorical(1, i, Long.MAX_VALUE, entries);
            try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
                assertNotNull(it);
                assertTrue("Not historical for iteration: " + i, it.historical(1));
                for (int j = i; j < entries; j++) {
                    assertTrue(it.hasNextX());
                    CacheDataRow row = it.next();
                    assertEquals(j * PARTS + 1, (int) row.key().value(grp.cacheObjectContext(), false));
                    assertEquals(j * PARTS + 1, (int) row.value().value(grp.cacheObjectContext(), false));
                }
                assertFalse(it.hasNext());
            }
        }
    } finally {
        stopAllGrids();
        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
    }
}

Example 9 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testWalAfterPreloading.

/**
 * @throws Exception If failed.
 */
public void testWalAfterPreloading() throws Exception {
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    try {
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        dbMgr.enableCheckpoints(false).get();
        int entries = 100;
        try (IgniteDataStreamer<Integer, Integer> streamer = ignite.dataStreamer(CACHE_NAME)) {
            for (int i = 0; i < entries; i++) streamer.addData(i, i);
        }
        IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
        for (int i = 0; i < entries; i++) assertEquals(new Integer(i), cache.get(i));
        stopGrid();
        ignite = startGrid();
        ignite.cluster().active(true);
        cache = ignite.cache(CACHE_NAME);
        for (int i = 0; i < entries; i++) assertEquals(new Integer(i), cache.get(i));
    } finally {
        stopAllGrids();
    }
}

Example 10 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method testWalTxSimple.

/**
 * @throws Exception If failed.
 */
public void testWalTxSimple() throws Exception {
    Ignite ignite = startGrid();
    ignite.cluster().active(true);
    try {
        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
        dbMgr.enableCheckpoints(false).get();
        IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
        int txCnt = 100;
        int keysPerTx = 10;
        for (int i = 0; i < txCnt; i++) {
            try (Transaction tx = ignite.transactions().txStart()) {
                for (int j = 0; j < keysPerTx; j++) {
                    int k = i * keysPerTx + j;
                    cache.put(k, new IndexedValue(k));
                }
                tx.commit();
            }
        }
        for (int i = 0; i < txCnt; i++) {
            for (int j = 0; j < keysPerTx; j++) {
                int k = i * keysPerTx + j;
                assertEquals(k, cache.get(k).value());
            }
        }
        stopGrid();
        ignite = startGrid();
        ignite.cluster().active(true);
        cache = ignite.cache(CACHE_NAME);
        for (int i = 0; i < txCnt; i++) {
            for (int j = 0; j < keysPerTx; j++) {
                int k = i * keysPerTx + j;
                assertEquals(k, cache.get(k).value());
            }
        }
        for (int i = 0; i < txCnt; i++) {
            for (int j = 0; j < keysPerTx; j++) {
                int k = i * keysPerTx + j;
                QueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
                List<List<?>> vals = cur.getAll();
                assertEquals(vals.size(), 1);
                assertEquals("string-" + k, vals.get(0).get(0));
            }
        }
    } finally {
        stopAllGrids();
    }
}