Examples with PageMemory org.apache.ignite.internal.pagemem.PageMemory used on opensource projects

Example 16 with PageMemory

use of org.apache.ignite.internal.pagemem.PageMemory in project ignite by apache.

the class CheckpointFailBeforeWriteMarkTest method testCheckpointFailBeforeMarkEntityWrite.

/**
 * @throws Exception if fail.
 */
@Test
public void testCheckpointFailBeforeMarkEntityWrite() throws Exception {
    // given: one node with persistence.
    IgniteEx ignite0 = startGrid(0);
    ignite0.cluster().active(true);
    // It is necessary to understanding when page replacement would be started.
    PageMemory pageMemory = ignite0.context().cache().context().database().dataRegion("default").pageMemory();
    // when: Load a lot of data to cluster.
    AtomicInteger lastKey = new AtomicInteger();
    GridTestUtils.runMultiThreadedAsync(() -> {
        IgniteCache<Integer, Object> cache2 = ignite(0).cache(DEFAULT_CACHE_NAME);
        // Should stopped putting data when node is fail.
        for (int i = 0; i < Integer.MAX_VALUE; i++) {
            cache2.put(i, i);
            lastKey.set(i);
            if (i % 1000 == 0)
                log.info("WRITE : " + i);
        }
    }, 3, "LOAD-DATA");
    // and: Page replacement was started.
    assertTrue(waitForCondition(() -> (int) U.field(pageMemory, "pageReplacementWarned") > 0, 60_000));
    // and: Node was failed during checkpoint after write lock was released and before checkpoint marker was stored to disk.
    interceptorIOFactory.triggerIOException((file) -> file.getName().contains("START.bin"));
    log.info("KILL NODE await to stop");
    assertTrue(waitForCondition(() -> G.allGrids().isEmpty(), 20_000));
    // then: Data recovery after node start should be successful.
    ignite0 = startGrid(0);
    ignite0.cluster().active(true);
    IgniteCache<Integer, Object> cache = ignite(0).cache(DEFAULT_CACHE_NAME);
    // WAL mode is 'default' so it is allowable to lost some last data(ex. last 100).
    for (int i = 0; i < lastKey.get() - 100; i++) assertNotNull(cache.get(i));
}

Example 17 with PageMemory

use of org.apache.ignite.internal.pagemem.PageMemory in project ignite by apache.

the class BPlusTreePageMemoryImplTest method createPageMemory.

/**
 * {@inheritDoc}
 */
@Override
protected PageMemory createPageMemory() throws Exception {
    long[] sizes = new long[CPUS + 1];
    for (int i = 0; i < sizes.length; i++) sizes[i] = 1024 * MB / CPUS;
    sizes[CPUS] = 10 * MB;
    DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setEncryptionSpi(new NoopEncryptionSpi());
    cfg.setMetricExporterSpi(new NoopMetricExporterSpi());
    cfg.setSystemViewExporterSpi(new JmxSystemViewExporterSpi());
    cfg.setDataStorageConfiguration(new DataStorageConfiguration());
    GridTestKernalContext cctx = new GridTestKernalContext(log, cfg);
    cctx.add(new IgnitePluginProcessor(cctx, cfg, Collections.emptyList()));
    cctx.add(new GridInternalSubscriptionProcessor(cctx));
    cctx.add(new PerformanceStatisticsProcessor(cctx));
    cctx.add(new GridEncryptionManager(cctx));
    cctx.add(new GridMetricManager(cctx));
    cctx.add(new GridSystemViewManager(cctx));
    GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(cctx, null, null, null, new NoOpPageStoreManager(), new NoOpWALManager(), null, new IgniteCacheDatabaseSharedManager(), null, null, null, null, null, null, null, null, null, null, null, null, new CacheDiagnosticManager());
    IgniteOutClosure<CheckpointProgress> clo = new IgniteOutClosure<CheckpointProgress>() {

        @Override
        public CheckpointProgress apply() {
            return Mockito.mock(CheckpointProgressImpl.class);
        }
    };
    PageMemory mem = new PageMemoryImpl(provider, sizes, sharedCtx, sharedCtx.pageStore(), PAGE_SIZE, (fullPageId, byteBuf, tag) -> {
        assert false : "No page replacement should happen during the test";
    }, new CIX3<Long, FullPageId, PageMemoryEx>() {

        @Override
        public void applyx(Long aLong, FullPageId fullPageId, PageMemoryEx ex) {
        }
    }, () -> true, new DataRegionMetricsImpl(new DataRegionConfiguration(), cctx), PageMemoryImpl.ThrottlingPolicy.DISABLED, clo);
    mem.start();
    return mem;
}

Example 18 with PageMemory

use of org.apache.ignite.internal.pagemem.PageMemory in project ignite by apache.

the class BPlusTreeReuseListPageMemoryImplTest method createPageMemory.

/**
 * {@inheritDoc}
 */
@Override
protected PageMemory createPageMemory() throws Exception {
    long[] sizes = new long[CPUS + 1];
    for (int i = 0; i < sizes.length; i++) sizes[i] = 1024 * MB / CPUS;
    sizes[CPUS] = 10 * MB;
    DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setEncryptionSpi(new NoopEncryptionSpi());
    cfg.setMetricExporterSpi(new NoopMetricExporterSpi());
    cfg.setSystemViewExporterSpi(new JmxSystemViewExporterSpi());
    cfg.setDataStorageConfiguration(new DataStorageConfiguration());
    GridTestKernalContext cctx = new GridTestKernalContext(log, cfg);
    cctx.add(new IgnitePluginProcessor(cctx, cfg, Collections.emptyList()));
    cctx.add(new GridInternalSubscriptionProcessor(cctx));
    cctx.add(new PerformanceStatisticsProcessor(cctx));
    cctx.add(new GridEncryptionManager(cctx));
    cctx.add(new GridMetricManager(cctx));
    cctx.add(new GridSystemViewManager(cctx));
    GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(cctx, null, null, null, new NoOpPageStoreManager(), new NoOpWALManager(), null, new IgniteCacheDatabaseSharedManager(), null, null, null, null, null, null, null, null, null, null, null, null, null);
    IgniteOutClosure<CheckpointProgress> clo = new IgniteOutClosure<CheckpointProgress>() {

        @Override
        public CheckpointProgress apply() {
            return Mockito.mock(CheckpointProgressImpl.class);
        }
    };
    PageMemory mem = new PageMemoryImpl(provider, sizes, sharedCtx, sharedCtx.pageStore(), PAGE_SIZE, (fullPageId, byteBuf, tag) -> {
        assert false : "No page replacement (rotation with disk) should happen during the test";
    }, new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {

        @Override
        public void applyx(Long page, FullPageId fullPageId, PageMemoryEx pageMem) {
        }
    }, () -> true, new DataRegionMetricsImpl(new DataRegionConfiguration(), cctx), PageMemoryImpl.ThrottlingPolicy.DISABLED, clo);
    mem.start();
    return mem;
}

Example 19 with PageMemory

use of org.apache.ignite.internal.pagemem.PageMemory in project ignite by apache.

the class PageMemoryNoStoreLeakTest method testPageDoubleInitMemoryLeak.

/**
 * @throws Exception If failed.
 */
@Test
public void testPageDoubleInitMemoryLeak() throws Exception {
    long initVMsize = D.getCommittedVirtualMemorySize();
    for (int i = 0; i < 1_000; i++) {
        final DirectMemoryProvider provider = new UnsafeMemoryProvider(log());
        final DataRegionConfiguration plcCfg = new DataRegionConfiguration().setMaxSize(MAX_MEMORY_SIZE).setInitialSize(MAX_MEMORY_SIZE);
        PageMemory mem = new PageMemoryNoStoreImpl(log(), provider, null, PAGE_SIZE, plcCfg, new DataRegionMetricsImpl(plcCfg, new GridTestKernalContext(log())), true);
        try {
            mem.start();
            // Second initialization, introduces leak
            mem.start();
        } finally {
            mem.stop(true);
        }
        long committedVMSize = D.getCommittedVirtualMemorySize();
        assertTrue(committedVMSize - initVMsize <= ALLOWED_DELTA);
    }
}

Example 20 with PageMemory

use of org.apache.ignite.internal.pagemem.PageMemory in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testGetForInitialWrite.

/**
 * @throws Exception if failed.
 */
@Test
public void testGetForInitialWrite() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
    int cacheId = shared.cache().cache(CACHE_NAME).context().cacheId();
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) shared.database();
    // Disable integrated checkpoint thread.
    dbMgr.enableCheckpoints(false);
    PageMemory mem = shared.database().dataRegion(null).pageMemory();
    IgniteWriteAheadLogManager wal = shared.wal();
    WALPointer start = wal.log(new CheckpointRecord(null));
    final FullPageId[] initWrites = new FullPageId[10];
    ig.context().cache().context().database().checkpointReadLock();
    try {
        for (int i = 0; i < initWrites.length; i++) initWrites[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
        // Check getForInitialWrite methods.
        for (FullPageId fullId : initWrites) {
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
                try {
                    DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.realPageSize(fullId.groupId()), null);
                    for (int i = PageIO.COMMON_HEADER_END + DataPageIO.ITEMS_OFF; i < mem.pageSize(); i++) PageUtils.putByte(pageAddr, i, (byte) 0xAB);
                    PageIO.printPage(pageAddr, mem.realPageSize(fullId.groupId()));
                } finally {
                    mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
                }
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
        wal.flush(null, false);
    } finally {
        ig.context().cache().context().database().checkpointReadUnlock();
        stopAllGrids(false);
    }
    ig = startGrid(0);
    ig.cluster().active(true);
    shared = ig.context().cache().context();
    dbMgr = (GridCacheDatabaseSharedManager) shared.database();
    dbMgr.enableCheckpoints(false);
    wal = shared.wal();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        it.next();
        for (FullPageId initialWrite : initWrites) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
            PageSnapshot snap = (PageSnapshot) tup.get2();
            FullPageId actual = snap.fullPageId();
            // there are extra tracking pages, skip them
            if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
                tup = it.next();
                assertTrue(tup.get2() instanceof PageSnapshot);
                actual = ((PageSnapshot) tup.get2()).fullPageId();
            }
            assertEquals(initialWrite, actual);
        }
    }
}