Examples with DataRegion org.apache.ignite.internal.processors.cache.persistence.DataRegion used on opensource projects

Example 6 with DataRegion

use of org.apache.ignite.internal.processors.cache.persistence.DataRegion in project ignite by apache.

the class IndexPagesMetricsInMemoryTest method validateIdxPagesCnt.

/**
 * {@inheritDoc}
 */
@Override
void validateIdxPagesCnt() throws IgniteCheckedException {
    DataRegion dataRegion = defaultDataRegion();
    long actualIdxPages = gridCacheProcessor().caches().stream().mapToInt(cache -> cache.context().groupId()).mapToObj(dataRegion.metrics()::cacheGrpPageMetrics).mapToLong(metrics -> metrics.indexPages().value()).sum();
    long expIdxPages = indexPageCounter.countIdxPagesInMemory(0);
    assertThat(actualIdxPages, is(expIdxPages));
    assertThat(dataRegion.metrics().pageMetrics().indexPages().value(), is(expIdxPages));
}

Example 7 with DataRegion

use of org.apache.ignite.internal.processors.cache.persistence.DataRegion in project ignite by apache.

the class CheckpointBufferDeadlockTest method runDeadlockScenario.

/**
 */
private void runDeadlockScenario() throws Exception {
    LogListener lsnr = LogListener.matches(s -> s.contains("AssertionError")).build();
    log.registerListener(lsnr);
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) ig.context().cache().context().database();
    FilePageStoreManager pageStoreMgr = (FilePageStoreManager) ig.context().cache().context().pageStore();
    final String cacheName = "single-part";
    CacheConfiguration<Object, Object> cacheCfg = new CacheConfiguration<>().setName(cacheName).setAffinity(new RendezvousAffinityFunction(false, 1));
    IgniteCache<Object, Object> singlePartCache = ig.getOrCreateCache(cacheCfg);
    db.enableCheckpoints(false).get();
    Thread.sleep(1_000);
    try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(singlePartCache.getName())) {
        int entries = MAX_SIZE / ENTRY_BYTE_CHUNK_SIZE / 4;
        for (int i = 0; i < entries; i++) streamer.addData(i, new byte[ENTRY_BYTE_CHUNK_SIZE]);
        streamer.flush();
    }
    slowCheckpointEnabled.set(true);
    log.info(">>> Slow checkpoints enabled");
    db.enableCheckpoints(true).get();
    AtomicBoolean fail = new AtomicBoolean(false);
    IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {

        @Override
        public void run() {
            int loops = 0;
            while (!stop.get()) {
                if (loops % 10 == 0 && loops > 0 && loops < 500 || loops % 500 == 0 && loops >= 500)
                    log.info("Successfully completed " + loops + " loops");
                db.checkpointReadLock();
                try {
                    Set<FullPageId> pickedPagesSet = new HashSet<>();
                    PageStore store = pageStoreMgr.getStore(CU.cacheId(cacheName), 0);
                    int pages = store.pages();
                    DataRegion region = db.dataRegion(DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME);
                    PageMemoryImpl pageMem = (PageMemoryImpl) region.pageMemory();
                    while (pickedPagesSet.size() < PAGES_TOUCHED_UNDER_CP_LOCK) {
                        int pageIdx = ThreadLocalRandom.current().nextInt(PAGES_TOUCHED_UNDER_CP_LOCK, pages - PAGES_TOUCHED_UNDER_CP_LOCK);
                        long pageId = PageIdUtils.pageId(0, PageIdAllocator.FLAG_DATA, pageIdx);
                        long page = pageMem.acquirePage(CU.cacheId(cacheName), pageId);
                        try {
                            // We do not know correct flag(FLAG_DATA or FLAG_AUX). Skip page if no luck.
                            if (pageId != PageIO.getPageId(page + PageMemoryImpl.PAGE_OVERHEAD))
                                continue;
                        } finally {
                            pageMem.releasePage(CU.cacheId(cacheName), pageId, page);
                        }
                        pickedPagesSet.add(new FullPageId(pageId, CU.cacheId(cacheName)));
                    }
                    List<FullPageId> pickedPages = new ArrayList<>(pickedPagesSet);
                    assertEquals(PAGES_TOUCHED_UNDER_CP_LOCK, pickedPages.size());
                    // Sort to avoid deadlocks on pages rw-locks.
                    pickedPages.sort(new Comparator<FullPageId>() {

                        @Override
                        public int compare(FullPageId o1, FullPageId o2) {
                            int cmp = Long.compare(o1.groupId(), o2.groupId());
                            if (cmp != 0)
                                return cmp;
                            return Long.compare(o1.effectivePageId(), o2.effectivePageId());
                        }
                    });
                    List<Long> readLockedPages = new ArrayList<>();
                    // Read lock many pages at once intentionally.
                    for (int i = 0; i < PAGES_TOUCHED_UNDER_CP_LOCK / 2; i++) {
                        FullPageId fpid = pickedPages.get(i);
                        long page = pageMem.acquirePage(fpid.groupId(), fpid.pageId());
                        long abs = pageMem.readLock(fpid.groupId(), fpid.pageId(), page);
                        assertFalse(fpid.toString(), abs == 0);
                        readLockedPages.add(page);
                    }
                    // Emulate writes to trigger throttling.
                    for (int i = PAGES_TOUCHED_UNDER_CP_LOCK / 2; i < PAGES_TOUCHED_UNDER_CP_LOCK && !stop.get(); i++) {
                        FullPageId fpid = pickedPages.get(i);
                        long page = pageMem.acquirePage(fpid.groupId(), fpid.pageId());
                        long abs = pageMem.writeLock(fpid.groupId(), fpid.pageId(), page);
                        assertFalse(fpid.toString(), abs == 0);
                        pageMem.writeUnlock(fpid.groupId(), fpid.pageId(), page, null, true);
                        pageMem.releasePage(fpid.groupId(), fpid.pageId(), page);
                    }
                    for (int i = 0; i < PAGES_TOUCHED_UNDER_CP_LOCK / 2; i++) {
                        FullPageId fpid = pickedPages.get(i);
                        pageMem.readUnlock(fpid.groupId(), fpid.pageId(), readLockedPages.get(i));
                        pageMem.releasePage(fpid.groupId(), fpid.pageId(), readLockedPages.get(i));
                    }
                } catch (Throwable e) {
                    log.error("Error in loader thread", e);
                    fail.set(true);
                } finally {
                    db.checkpointReadUnlock();
                }
                loops++;
            }
        }
    }, 10, "load-runner");
    // Await for the start of throttling.
    Thread.sleep(10_000);
    slowCheckpointEnabled.set(false);
    log.info(">>> Slow checkpoints disabled");
    assertFalse(fail.get());
    // Previous checkpoint should eventually finish.
    forceCheckpoint();
    stop.set(true);
    fut.get();
    db.enableCheckpoints(true).get();
    // check that there is no problem with pinned pages
    ig.destroyCache(cacheName);
    assertFalse(lsnr.check());
    log.unregisterListener(lsnr);
}

Example 8 with DataRegion

use of org.apache.ignite.internal.processors.cache.persistence.DataRegion in project ignite by apache.

the class SwapPathConstructionSelfTest method extractDefaultPageMemoryAllocPath.

/**
 * @param context Context.
 */
private String extractDefaultPageMemoryAllocPath(GridKernalContext context) {
    IgniteCacheDatabaseSharedManager dbMgr = context.cache().context().database();
    Map<String, DataRegion> memPlcMap = U.field(dbMgr, "dataRegionMap");
    PageMemory pageMem = memPlcMap.get("default").pageMemory();
    Object memProvider = U.field(pageMem, "directMemoryProvider");
    Object memProvider0 = U.field(memProvider, "memProvider");
    return ((File) U.field(memProvider0, "allocationPath")).getAbsolutePath();
}

Example 9 with DataRegion

use of org.apache.ignite.internal.processors.cache.persistence.DataRegion in project ignite by apache.

the class IndexPagesMetricsPageDisplacementTest method testPageDisplacement.

/**
 * Inserts data into the cache as long as it fills enough for some data pages to get dumped to the
 * storage. Since index page metrics should reflect the number of in-memory pages, the metric value is expected to
 * go down.
 */
@Test
public void testPageDisplacement() throws IgniteCheckedException {
    int grpId = grid.cachex(TEST_CACHE_NAME).context().groupId();
    DataRegion dataRegion = grid.context().cache().context().database().dataRegion(null);
    PageMetrics pageMetrics = dataRegion.metrics().cacheGrpPageMetrics(grpId);
    int personId = 0;
    long idxPagesOnDisk;
    long idxPagesInMemory;
    do {
        // insert data into the cache until some index pages get displaced to the storage
        for (int i = 0; i < 100; i++, personId++) cache.put(personId, new Person(personId, "foobar"));
        forceCheckpoint(grid);
        idxPagesOnDisk = getIdxPagesOnDisk(grpId).size();
        idxPagesInMemory = idxPageCounter.countIdxPagesInMemory(grpId);
        assertThat(pageMetrics.indexPages().value(), is(idxPagesInMemory));
    } while (idxPagesOnDisk <= idxPagesInMemory);
    // load pages back into memory and check that the metric value has increased
    touchIdxPages(grpId);
    long allIdxPagesInMemory = idxPageCounter.countIdxPagesInMemory(grpId);
    assertThat(allIdxPagesInMemory, greaterThan(idxPagesInMemory));
    assertThat(pageMetrics.indexPages().value(), is(allIdxPagesInMemory));
}

Example 10 with DataRegion

use of org.apache.ignite.internal.processors.cache.persistence.DataRegion in project ignite by apache.

the class LoadAllWarmUpStrategySelfTest method testMemoryLessPds.

/**
 * Test checks that if memory is less than pds, not all pages in pds will warm-up.
 * There may be evictions during warm-up, so count of pages loaded is not maximum.
 * <p/>
 * Steps:
 * 1)Start node and fill it with data for first data region until it is 2 * {@code MIN_PAGE_MEMORY_SIZE};
 * 2)Make a checkpoint;
 * 3)Restart node with warm-up, change maximum data region size to {@code MIN_PAGE_MEMORY_SIZE},
 * and listen for {@link LoadAllWarmUpStrategyEx#loadDataInfo};
 * 4)Check that estimated count of pages to warm-up is between maximum and
 * approximate minimum count of pages to load;
 * 5)Checking that total count of pages loaded is between maximum and
 * approximate minimum count of pages to load.
 *
 * Approximate value due to fact that there are already loaded pages at
 * beginning of warm-up, as well as evictions occur during warm-up.
 *
 * @throws Exception If failed.
 */
@Test
public void testMemoryLessPds() throws Exception {
    IgniteEx n = startGrid(0);
    n.cluster().state(ClusterState.ACTIVE);
    int i = 0;
    final long minMemSize = U.field(IgniteCacheDatabaseSharedManager.class, "MIN_PAGE_MEMORY_SIZE");
    DataRegion dr_0 = n.context().cache().context().database().dataRegion("dr_0");
    while (dr_0.pageMemory().loadedPages() * dr_0.pageMemory().systemPageSize() < 2 * minMemSize) {
        n.cache("c_0").put("c_0" + i, new Organization(i, "c_0" + i));
        n.cache("c_1").put("c_1" + i, new Person(i, "c_1" + i, i));
        i++;
    }
    forceCheckpoint();
    stopAllGrids();
    warmUp = true;
    IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(0));
    cfg.getDataStorageConfiguration().getDataRegionConfigurations()[0].setMaxSize(minMemSize);
    Map<String, Map<CacheGroupContext, List<LoadPartition>>> loadDataInfoMap = new ConcurrentHashMap<>();
    LoadAllWarmUpStrategyEx.loadDataInfoCb = loadDataInfoMap::put;
    n = startGrid(cfg);
    dr_0 = n.context().cache().context().database().dataRegion("dr_0");
    long warmUpPageCnt = loadDataInfoMap.get("dr_0").values().stream().flatMap(Collection::stream).mapToLong(LoadPartition::pages).sum();
    long maxLoadPages = minMemSize / dr_0.pageMemory().systemPageSize();
    long minLoadPages = maxLoadPages - 100;
    long loadPages = dr_0.pageMemory().loadedPages();
    // There are loaded pages before warm-up.
    assertTrue(warmUpPageCnt >= minLoadPages && warmUpPageCnt <= maxLoadPages);
    // Pages may be evicted.
    assertTrue(loadPages >= minLoadPages && loadPages <= maxLoadPages);
}