Examples with GridFinishedFuture org.apache.ignite.internal.util.future.GridFinishedFuture used on opensource projects

Example 31 with GridFinishedFuture

use of org.apache.ignite.internal.util.future.GridFinishedFuture in project ignite by apache.

the class PageMemoryImplTest method runThrottlingEmptifyCpBufFirst.

/**
 * @throws Exception if failed.
 */
public void runThrottlingEmptifyCpBufFirst(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
    TestPageStoreManager pageStoreMgr = new TestPageStoreManager();
    final List<FullPageId> allocated = new ArrayList<>();
    int pagesForStartThrottling = 10;
    // Number of pages which were poll from checkpoint buffer for throttling.
    AtomicInteger cpBufferPollPages = new AtomicInteger();
    // Create a 1 mb page memory.
    PageMemoryImpl memory = createPageMemory(1, plc, pageStoreMgr, pageStoreMgr, (IgniteInClosure<FullPageId>) fullPageId -> {
        assertEquals(cpBufferPollPages.incrementAndGet(), pageStoreMgr.storedPages.size());
    });
    assert pagesForStartThrottling < memory.checkpointBufferPagesSize() / 3;
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) {
        long id = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
        FullPageId fullId = new FullPageId(id, 1);
        allocated.add(fullId);
        writePage(memory, fullId, (byte) 1);
    }
    GridMultiCollectionWrapper<FullPageId> markedPages = memory.beginCheckpoint(new GridFinishedFuture());
    for (int i = 0; i < pagesForStartThrottling + (memory.checkpointBufferPagesSize() * 2 / 3); i++) writePage(memory, allocated.get(i), (byte) 1);
    doCheckpoint(markedPages, memory, pageStoreMgr);
    // There is 'pagesForStartThrottling - 1' because we should write pagesForStartThrottling pages
    // from checkpoint buffer before throttling will be disabled but at least one page always would be written
    // outside of throttling and in our case we certainly know that this page is also contained in checkpoint buffer
    // (because all of our pages are in checkpoint buffer).
    assertEquals(pagesForStartThrottling - 1, cpBufferPollPages.get());
}

Example 32 with GridFinishedFuture

use of org.apache.ignite.internal.util.future.GridFinishedFuture in project ignite by apache.

the class PageMemoryImplTest method testCheckpointBufferCantOverflowWithThrottlingMixedLoad.

/**
 * @throws Exception If failed.
 */
private void testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
    PageMemoryImpl memory = createPageMemory(plc, null);
    List<FullPageId> pages = new ArrayList<>();
    for (int i = 0; i < (MAX_SIZE - 10) * MB / PAGE_SIZE / 2; i++) {
        long pageId = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
        FullPageId fullPageId = new FullPageId(pageId, 1);
        pages.add(fullPageId);
        acquireAndReleaseWriteLock(memory, fullPageId);
    }
    memory.beginCheckpoint(new GridFinishedFuture());
    CheckpointMetricsTracker mockTracker = Mockito.mock(CheckpointMetricsTracker.class);
    for (FullPageId checkpointPage : pages) memory.checkpointWritePage(checkpointPage, ByteBuffer.allocate(PAGE_SIZE), (fullPageId, buffer, tag) -> {
    // No-op.
    }, mockTracker);
    memory.finishCheckpoint();
    for (int i = (int) ((MAX_SIZE - 10) * MB / PAGE_SIZE / 2); i < (MAX_SIZE - 20) * MB / PAGE_SIZE; i++) {
        long pageId = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
        FullPageId fullPageId = new FullPageId(pageId, 1);
        pages.add(fullPageId);
        acquireAndReleaseWriteLock(memory, fullPageId);
    }
    memory.beginCheckpoint(new GridFinishedFuture());
    // Mix pages in checkpoint with clean pages
    Collections.shuffle(pages);
    AtomicBoolean stop = new AtomicBoolean(false);
    try {
        GridTestUtils.runAsync(() -> {
            for (FullPageId page : pages) {
                if (// Mark dirty 50% of pages
                ThreadLocalRandom.current().nextDouble() < 0.5)
                    try {
                        acquireAndReleaseWriteLock(memory, page);
                        if (stop.get())
                            break;
                    } catch (IgniteCheckedException e) {
                        log.error("runAsync ended with exception", e);
                        fail();
                    }
            }
        }).get(5_000);
    } catch (IgniteFutureTimeoutCheckedException ignore) {
    // Expected.
    } finally {
        stop.set(true);
    }
    memory.finishCheckpoint();
    LongAdderMetric totalThrottlingTime = U.field(memory.metrics(), "totalThrottlingTime");
    assertNotNull(totalThrottlingTime);
    assertTrue(totalThrottlingTime.value() > 0);
}

Example 33 with GridFinishedFuture

use of org.apache.ignite.internal.util.future.GridFinishedFuture in project ignite by apache.

the class PageMemoryImplTest method testCheckpointBufferOverusageDontCauseWriteLockLeak.

/**
 * @throws Exception If failed.
 */
@Test
public void testCheckpointBufferOverusageDontCauseWriteLockLeak() throws Exception {
    PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED, null);
    List<FullPageId> pages = new ArrayList<>();
    try {
        while (!Thread.currentThread().isInterrupted()) {
            long pageId = memory.allocatePage(1, INDEX_PARTITION, FLAG_IDX);
            FullPageId fullPageId = new FullPageId(pageId, 1);
            pages.add(fullPageId);
            // to set page id, otherwise we would fail with assertion error
            acquireAndReleaseWriteLock(memory, fullPageId);
        }
    } catch (IgniteOutOfMemoryException ignore) {
    // Success
    }
    memory.beginCheckpoint(new GridFinishedFuture());
    final AtomicReference<FullPageId> lastPage = new AtomicReference<>();
    try {
        for (FullPageId fullPageId : pages) {
            lastPage.set(fullPageId);
            acquireAndReleaseWriteLock(memory, fullPageId);
        }
    } catch (Exception ex) {
        assertTrue(ex.getMessage().startsWith(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG));
    }
    memory.finishCheckpoint();
    GridTestUtils.runAsync(() -> {
        try {
            // we should be able get lock again
            acquireAndReleaseWriteLock(memory, lastPage.get());
        } catch (IgniteCheckedException e) {
            throw new AssertionError(e);
        }
    }).get(getTestTimeout());
}

Example 34 with GridFinishedFuture

use of org.apache.ignite.internal.util.future.GridFinishedFuture in project ignite by apache.

the class IgnitePageMemReplaceDelayedWriteUnitTest method testBackwardCompatibilityMode.

/**
 * Test delayed eviction causes locking in page reads
 * @throws IgniteCheckedException if failed.
 */
@Test
public void testBackwardCompatibilityMode() throws IgniteCheckedException {
    IgniteConfiguration cfg = getConfiguration(16 * MB);
    AtomicInteger totalEvicted = new AtomicInteger();
    PageStoreWriter pageWriter = (FullPageId fullPageId, ByteBuffer byteBuf, int tag) -> {
        log.info("Evicting " + fullPageId);
        assert getSegment(fullPageId).writeLock().isHeldByCurrentThread();
        totalEvicted.incrementAndGet();
    };
    System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "false");
    int pageSize = 4096;
    PageMemoryImpl memory;
    try {
        memory = createPageMemory(cfg, pageWriter, pageSize);
    } finally {
        System.clearProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE);
    }
    this.pageMemory = memory;
    long pagesTotal = cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().getMaxSize() / pageSize;
    long markDirty = pagesTotal * 2 / 3;
    for (int i = 0; i < markDirty; i++) {
        long pageId = memory.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
        long ptr = memory.acquirePage(1, pageId);
        memory.releasePage(1, pageId, ptr);
    }
    GridMultiCollectionWrapper<FullPageId> ids = memory.beginCheckpoint(new GridFinishedFuture());
    int cpPages = ids.size();
    log.info("Started CP with [" + cpPages + "] pages in it, created [" + markDirty + "] pages");
    for (int i = 0; i < cpPages; i++) {
        long pageId = memory.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
        long ptr = memory.acquirePage(1, pageId);
        memory.releasePage(1, pageId, ptr);
    }
    assert totalEvicted.get() > 0;
    memory.stop(true);
}

Example 35 with GridFinishedFuture

use of org.apache.ignite.internal.util.future.GridFinishedFuture in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testDirtyFlag.

/**
 * @throws Exception if failed.
 */
@Test
public void testDirtyFlag() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
    int cacheId = shared.cache().cache(CACHE_NAME).context().cacheId();
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) shared.database();
    // Disable integrated checkpoint thread.
    dbMgr.enableCheckpoints(false).get();
    PageMemoryEx mem = (PageMemoryEx) dbMgr.dataRegion(null).pageMemory();
    FullPageId[] pageIds = new FullPageId[100];
    DummyPageIO pageIO = new DummyPageIO();
    ig.context().cache().context().database().checkpointReadLock();
    try {
        for (int i = 0; i < pageIds.length; i++) pageIds[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
        for (FullPageId fullId : pageIds) {
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                // page is dirty right after allocation
                assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
                long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
                try {
                    pageIO.initNewPage(pageAddr, fullId.pageId(), mem.realPageSize(fullId.groupId()), null);
                    assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
                } finally {
                    mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
                }
                assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
    } finally {
        ig.context().cache().context().database().checkpointReadUnlock();
    }
    Collection<FullPageId> cpPages = mem.beginCheckpoint(new GridFinishedFuture());
    ig.context().cache().context().database().checkpointReadLock();
    try {
        for (FullPageId fullId : pageIds) {
            assertTrue(cpPages.contains(fullId));
            ByteBuffer buf = ByteBuffer.allocate(mem.pageSize());
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
                long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
                try {
                    assertFalse(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
                    for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) PageUtils.putByte(pageAddr, i, (byte) 1);
                } finally {
                    mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
                }
                assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
                buf.rewind();
                mem.checkpointWritePage(fullId, buf, (fullPageId, buffer, tag) -> {
                }, null);
                buf.position(PageIO.COMMON_HEADER_END);
                while (buf.hasRemaining()) assertEquals((byte) 0, buf.get());
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
    } finally {
        ig.context().cache().context().database().checkpointReadUnlock();
    }
    mem.finishCheckpoint();
    for (FullPageId fullId : pageIds) {
        long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
        try {
            assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
        } finally {
            mem.releasePage(fullId.groupId(), fullId.pageId(), page);
        }
    }
}