Examples with WALPointer org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer used on opensource projects

Example 86 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class CheckpointHistory method calculateWalSegmentsCovered.

/**
 * Calculates indexes of WAL segments covered by last checkpoint.
 *
 * @return list of indexes or empty list if there are no checkpoints.
 */
private IgniteBiTuple<Long, Long> calculateWalSegmentsCovered() {
    IgniteBiTuple<Long, Long> tup = new IgniteBiTuple<>(-1L, -1L);
    Map.Entry<Long, CheckpointEntry> lastEntry = histMap.lastEntry();
    if (lastEntry == null)
        return tup;
    Map.Entry<Long, CheckpointEntry> previousEntry = histMap.lowerEntry(lastEntry.getKey());
    WALPointer lastWALPointer = lastEntry.getValue().checkpointMark();
    long lastIdx = lastWALPointer.index();
    long prevIdx = 0;
    if (previousEntry != null)
        prevIdx = previousEntry.getValue().checkpointMark().index();
    tup.set1(prevIdx);
    tup.set2(lastIdx - 1);
    return tup;
}

Example 87 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class CdcCacheVersionTest method testOrderIncrease.

/**
 */
@Test
public void testOrderIncrease() throws Exception {
    walProvider = (ctx) -> new FileWriteAheadLogManager(ctx) {

        /**
         */
        private long prevOrder = -1;

        @Override
        public WALPointer log(WALRecord rec) throws IgniteCheckedException {
            if (rec.type() != DATA_RECORD_V2)
                return super.log(rec);
            DataRecord dataRec = (DataRecord) rec;
            for (int i = 0; i < dataRec.entryCount(); i++) {
                assertEquals(CU.cacheId(DEFAULT_CACHE_NAME), dataRec.get(i).cacheId());
                assertEquals(KEY_TO_UPD, (int) dataRec.get(i).key().value(null, false));
                assertTrue(dataRec.get(i).writeVersion().order() > prevOrder);
                prevOrder = dataRec.get(i).writeVersion().order();
                walRecCheckedCntr.incrementAndGet();
            }
            return super.log(rec);
        }
    };
    IgniteConfiguration cfg = getConfiguration("ignite-0");
    IgniteEx ign = startGrid(cfg);
    ign.cluster().state(ACTIVE);
    IgniteCache<Integer, User> cache = ign.getOrCreateCache(new CacheConfiguration<Integer, User>(DEFAULT_CACHE_NAME).setAtomicityMode(atomicityMode).setCacheMode(cacheMode));
    walRecCheckedCntr.set(0);
    // Expect {@link CacheEntryVersion#order()} will monotically increase.
    for (int i = 0; i < KEYS_CNT; i++) cache.put(KEY_TO_UPD, createUser(i));
    assertTrue(waitForCondition(() -> walRecCheckedCntr.get() == KEYS_CNT, getTestTimeout()));
}

Example 88 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgnitePdsRecoveryAfterFileCorruptionTest method generateWal.

/**
 * @param mem Mem.
 * @param storeMgr Store manager.
 * @param wal Wal.
 * @param cacheId Cache id.
 * @param pages Pages.
 */
private void generateWal(final PageMemoryImpl mem, final IgnitePageStoreManager storeMgr, final IgniteWriteAheadLogManager wal, final int cacheId, FullPageId[] pages) throws Exception {
    // Mark the start position.
    CheckpointRecord cpRec = new CheckpointRecord(null);
    WALPointer start = wal.log(cpRec);
    wal.flush(start, false);
    for (FullPageId fullId : pages) {
        long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
        try {
            long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
            PageIO.setPageId(pageAddr, fullId.pageId());
            try {
                for (int j = PageIO.COMMON_HEADER_END; j < mem.realPageSize(fullId.groupId()); j += 4) PageUtils.putInt(pageAddr, j, j + (int) fullId.pageId());
            } finally {
                mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
            }
        } finally {
            mem.releasePage(fullId.groupId(), fullId.pageId(), page);
        }
    }
    Collection<FullPageId> pageIds = mem.beginCheckpoint(new GridFinishedFuture());
    info("Acquired pages for checkpoint: " + pageIds.size());
    try {
        long begin = System.currentTimeMillis();
        long cp = 0;
        AtomicLong write = new AtomicLong();
        PageStoreWriter pageStoreWriter = (fullPageId, buf, tag) -> {
            int groupId = fullPageId.groupId();
            long pageId = fullPageId.pageId();
            for (int j = PageIO.COMMON_HEADER_END; j < mem.realPageSize(groupId); j += 4) assertEquals(j + (int) pageId, buf.getInt(j));
            buf.rewind();
            long writeStart = System.nanoTime();
            storeMgr.write(cacheId, pageId, buf, tag, true);
            long writeEnd = System.nanoTime();
            write.getAndAdd(writeEnd - writeStart);
        };
        ByteBuffer tmpBuf = ByteBuffer.allocate(mem.pageSize());
        tmpBuf.order(ByteOrder.nativeOrder());
        for (FullPageId fullId : pages) {
            if (pageIds.contains(fullId)) {
                long cpStart = System.nanoTime();
                mem.checkpointWritePage(fullId, tmpBuf, pageStoreWriter, null);
                long cpEnd = System.nanoTime();
                cp += cpEnd - cpStart;
            }
        }
        long syncStart = System.currentTimeMillis();
        storeMgr.sync(cacheId, 0);
        long end = System.currentTimeMillis();
        info("Written pages in " + (end - begin) + "ms, copy took " + (cp / 1_000_000) + "ms, " + "write took " + (write.get() / 1_000_000) + "ms, sync took " + (end - syncStart) + "ms");
    } finally {
        info("Finishing checkpoint...");
        mem.finishCheckpoint();
        info("Finished checkpoint");
    }
    wal.flush(wal.log(new CheckpointRecord(null)), false);
    for (FullPageId fullId : pages) {
        long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
        try {
            assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page));
            assertFalse("Page is dirty after the last checkpoint: [cacheId=" + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.groupId(), fullId.pageId(), page));
        } finally {
            mem.releasePage(fullId.groupId(), fullId.pageId(), page);
        }
    }
}

Example 89 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class LocalWalModeNoChangeDuringRebalanceOnNonNodeAssignTest method check.

/**
 * @throws Exception If failed.
 */
public void check() throws Exception {
    Ignite ig = startGridsMultiThreaded(NODES);
    int entries = 100_000;
    try (IgniteDataStreamer<Integer, Integer> st = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
        st.allowOverwrite(true);
        for (int i = 0; i < entries; i++) st.addData(i, -i);
    }
    IgniteEx ig4 = startGrid(NODES);
    ig4.cluster().setBaselineTopology(ig4.context().discovery().topologyVersion());
    IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory(log);
    String name = ig4.name();
    try (WALIterator it = iteratorFactory.iterator(walPath(name), walArchivePath(name))) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec.type() == METASTORE_DATA_RECORD) {
                MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord) rec;
                String key = metastoreDataRecord.key();
                if (key.startsWith("grp-wal-") && key.contains(valueOf(cacheId(DEFAULT_CACHE_NAME))) && metastoreDataRecord.value() != null)
                    fail("WAL was disabled but should not.");
            }
        }
    }
}

Example 90 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class ReleaseSegmentOnHistoricalRebalanceTest method testReleaseDuringRebalanceIterator.

/**
 * Checks that if release the segment during {@link IgniteCacheOffheapManagerImpl#rebalanceIterator},
 * there will be no errors and the rebalance will be completed.
 *
 * @throws Exception If failed.
 */
@Test
public void testReleaseDuringRebalanceIterator() throws Exception {
    checkHistoricalRebalance(n -> {
        IgniteInternalCache<?, ?> cachex = n.cachex(DEFAULT_CACHE_NAME);
        GridCacheOffheapManager spy = spy(offheapManager(cachex));
        doAnswer(m -> {
            WALPointer reserved = dbMgr(n).latestWalPointerReservedForPreloading();
            assertNotNull(reserved);
            Object o = m.callRealMethod();
            release(n, reserved);
            assertTrue(segmentAware(n).minReserveIndex(Long.MAX_VALUE));
            return o;
        }).when(spy).rebalanceIterator(any(), any());
        offheapManager(cachex, spy);
    });
}