Examples with CheckpointRecord org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord used on opensource projects

Example 11 with CheckpointRecord

use of org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testGetForInitialWrite.

/**
 * @throws Exception if failed.
 */
@Test
public void testGetForInitialWrite() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
    int cacheId = shared.cache().cache(CACHE_NAME).context().cacheId();
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) shared.database();
    // Disable integrated checkpoint thread.
    dbMgr.enableCheckpoints(false);
    PageMemory mem = shared.database().dataRegion(null).pageMemory();
    IgniteWriteAheadLogManager wal = shared.wal();
    WALPointer start = wal.log(new CheckpointRecord(null));
    final FullPageId[] initWrites = new FullPageId[10];
    ig.context().cache().context().database().checkpointReadLock();
    try {
        for (int i = 0; i < initWrites.length; i++) initWrites[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
        // Check getForInitialWrite methods.
        for (FullPageId fullId : initWrites) {
            long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
            try {
                long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
                try {
                    DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.realPageSize(fullId.groupId()), null);
                    for (int i = PageIO.COMMON_HEADER_END + DataPageIO.ITEMS_OFF; i < mem.pageSize(); i++) PageUtils.putByte(pageAddr, i, (byte) 0xAB);
                    PageIO.printPage(pageAddr, mem.realPageSize(fullId.groupId()));
                } finally {
                    mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
                }
            } finally {
                mem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        }
        wal.flush(null, false);
    } finally {
        ig.context().cache().context().database().checkpointReadUnlock();
        stopAllGrids(false);
    }
    ig = startGrid(0);
    ig.cluster().active(true);
    shared = ig.context().cache().context();
    dbMgr = (GridCacheDatabaseSharedManager) shared.database();
    dbMgr.enableCheckpoints(false);
    wal = shared.wal();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        it.next();
        for (FullPageId initialWrite : initWrites) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
            PageSnapshot snap = (PageSnapshot) tup.get2();
            FullPageId actual = snap.fullPageId();
            // there are extra tracking pages, skip them
            if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
                tup = it.next();
                assertTrue(tup.get2() instanceof PageSnapshot);
                actual = ((PageSnapshot) tup.get2()).fullPageId();
            }
            assertEquals(initialWrite, actual);
        }
    }
}

Example 12 with CheckpointRecord

use of org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method checkDataWalEntries.

/**
 * @throws Exception if failed.
 */
private void checkDataWalEntries(boolean mvcc) throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
    GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(mvcc ? MVCC_CACHE_NAME : CACHE_NAME).context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    IgniteWriteAheadLogManager wal = sharedCtx.wal();
    assertTrue(wal.isAlwaysWriteFullPages());
    db.enableCheckpoints(false).get();
    final int cnt = 10;
    List<DataEntry> entries = new ArrayList<>(cnt);
    for (int i = 0; i < cnt; i++) {
        GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
        KeyCacheObject key = cctx.toCacheKeyObject(i);
        CacheObject val = null;
        if (op != GridCacheOperation.DELETE)
            val = cctx.toCacheObject("value-" + i);
        entries.add(mvcc ? new MvccDataEntry(cctx.cacheId(), key, val, op, null, cctx.cache().nextVersion(), 0L, cctx.affinity().partition(i), i, new MvccVersionImpl(1000L, 10L, i + 1)) : new DataEntry(cctx.cacheId(), key, val, op, null, cctx.cache().nextVersion(), 0L, cctx.affinity().partition(i), i, DataEntry.EMPTY_FLAGS));
    }
    UUID cpId = UUID.randomUUID();
    WALPointer start = wal.log(new CheckpointRecord(cpId, null));
    wal.flush(start, false);
    for (DataEntry entry : entries) wal.log(mvcc ? new MvccDataRecord((MvccDataEntry) entry) : new DataRecord(entry));
    // Data will not be written to the page store.
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().active(true);
    sharedCtx = ig.context().cache().context();
    cctx = sharedCtx.cache().cache(mvcc ? MVCC_CACHE_NAME : CACHE_NAME).context();
    db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    wal = sharedCtx.wal();
    db.enableCheckpoints(false).get();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
        assert cpRecordTup.get2() instanceof CheckpointRecord;
        assertEquals(start, cpRecordTup.get1());
        CheckpointRecord cpRec = (CheckpointRecord) cpRecordTup.get2();
        assertEquals(cpId, cpRec.checkpointId());
        assertNull(cpRec.checkpointMark());
        assertFalse(cpRec.end());
        int idx = 0;
        CacheObjectContext coctx = cctx.cacheObjectContext();
        while (idx < entries.size()) {
            IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
            if (!mvcc)
                assert dataRecTup.get2() instanceof DataRecord;
            else
                assert dataRecTup.get2() instanceof MvccDataRecord;
            DataRecord dataRec = (DataRecord) dataRecTup.get2();
            DataEntry entry = entries.get(idx);
            assertEquals(1, dataRec.entryCount());
            DataEntry readEntry = dataRec.get(0);
            assertEquals(entry.cacheId(), readEntry.cacheId());
            assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
            assertEquals(entry.op(), readEntry.op());
            if (entry.op() == GridCacheOperation.UPDATE)
                assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
            else
                assertNull(entry.value());
            assertEquals(entry.writeVersion(), readEntry.writeVersion());
            assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
            assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
            if (mvcc) {
                assert entry instanceof MvccDataEntry;
                assert readEntry instanceof MvccDataEntry;
                assertEquals(((MvccDataEntry) entry).mvccVer(), ((MvccDataEntry) readEntry).mvccVer());
            }
            idx++;
        }
    }
}

Example 13 with CheckpointRecord

use of org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method verifyReads.

/**
 * @param res Result map to verify.
 * @param mem Memory.
 */
private void verifyReads(GridKernalContext ctx, Map<FullPageId, Integer> res, PageMemory mem, WALPointer start, IgniteWriteAheadLogManager wal) throws Exception {
    Map<FullPageId, byte[]> replay = new HashMap<>();
    ByteBuffer buf = ByteBuffer.allocateDirect(mem.pageSize()).order(ByteOrder.nativeOrder());
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
        assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
        CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
        while (it.hasNext()) {
            tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof CheckpointRecord) {
                CheckpointRecord end = (CheckpointRecord) rec;
                // Found the finish mark.
                if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
                    break;
            } else if (rec instanceof PageSnapshot) {
                PageSnapshot page = (PageSnapshot) rec;
                int realPageSize = mem.realPageSize(page.groupId());
                byte[] pageData = page.pageData();
                if (page.pageDataSize() < realPageSize) {
                    buf.clear();
                    buf.put(pageData).flip();
                    ctx.compress().decompressPage(buf, realPageSize);
                    pageData = new byte[buf.limit()];
                    buf.get(pageData);
                }
                replay.put(page.fullPageId(), pageData);
            }
        }
    }
    // Check read-through from the file store.
    for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
        FullPageId fullId = entry.getKey();
        int state = entry.getValue();
        if (state == -1) {
            info("Page was never written: " + fullId);
            continue;
        }
        byte[] walData = replay.get(fullId);
        assertNotNull("Missing WAL record for a written page: " + fullId, walData);
        long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
        try {
            long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
            try {
                for (int i = PageIO.COMMON_HEADER_END; i < mem.realPageSize(fullId.groupId()); i++) {
                    int expState = state & 0xFF;
                    int pageState = PageUtils.getByte(pageAddr, i) & 0xFF;
                    int walState = walData[i] & 0xFF;
                    if (expState != pageState)
                        assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']', expState, pageState);
                    if (expState != walState)
                        assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']', expState, walState);
                }
            } finally {
                mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
            }
        } finally {
            mem.releasePage(fullId.groupId(), fullId.pageId(), page);
        }
    }
}

Example 14 with CheckpointRecord

use of org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord in project ignite by apache.

the class IgnitePdsRecoveryAfterFileCorruptionTest method testPageRecoveryAfterFileCorruption.

/**
 * @throws Exception if failed.
 */
@Test
public void testPageRecoveryAfterFileCorruption() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
    // Put for create data store and init meta page.
    cache.put(1, 1);
    GridCacheSharedContext sharedCtx = ig.context().cache().context();
    GridCacheDatabaseSharedManager psMgr = (GridCacheDatabaseSharedManager) sharedCtx.database();
    FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
    U.sleep(1_000);
    // Disable integrated checkpoint thread.
    psMgr.enableCheckpoints(false).get();
    PageMemory mem = sharedCtx.database().dataRegion(policyName).pageMemory();
    DummyPageIO pageIO = new DummyPageIO();
    int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
    int pagesCnt = getTotalPagesToTest();
    FullPageId[] pages = new FullPageId[pagesCnt];
    // Get lock to prevent assertion. A new page should be allocated under checkpoint lock.
    psMgr.checkpointReadLock();
    try {
        for (int i = 0; i < pagesCnt; i++) {
            pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
            initPage(mem, pageIO, pages[i]);
        }
        generateWal((PageMemoryImpl) mem, sharedCtx.pageStore(), sharedCtx.wal(), cacheId, pages);
    } finally {
        psMgr.checkpointReadUnlock();
    }
    eraseDataFromDisk(pageStore, cacheId, pages[0]);
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    checkRestore(ig, pages);
    // It is necessary to clear the current WAL history to make sure that the restored pages have been saved.
    GridCacheSharedContext<Object, Object> cctx = ig.context().cache().context();
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) cctx.database();
    FileWriteAheadLogManager wal = (FileWriteAheadLogManager) cctx.wal();
    // Force checkpoint.
    dbMgr.enableCheckpoints(true).get(getTestTimeout());
    dbMgr.checkpointReadLock();
    try {
        WALPointer lastWalPtr = dbMgr.checkpointHistory().lastCheckpoint().checkpointMark();
        // Move current WAL segment into the archive.
        wal.log(new CheckpointRecord(null), RolloverType.NEXT_SEGMENT);
        assertTrue(waitForCondition(() -> wal.lastArchivedSegment() >= lastWalPtr.index(), getTestTimeout()));
        wal.truncate(lastWalPtr);
        dbMgr.onWalTruncated(lastWalPtr);
    } finally {
        dbMgr.checkpointReadUnlock();
    }
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    checkRestore(ig, pages);
}

Example 15 with CheckpointRecord

use of org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord in project ignite by apache.

the class WalScannerTest method shouldFindCorrectRecordsForMoreThanOnePages.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldFindCorrectRecordsForMoreThanOnePages() throws Exception {
    // given: Iterator with random value and value which should be find by scanner with several ids.
    long expPageId1 = 984;
    long expPageId2 = 9584;
    long expPageId3 = 98344;
    int grpId = 123;
    PageSnapshot expPageSnapshot = new PageSnapshot(new FullPageId(expPageId1, grpId), dummyPage(1024, expPageId1), 1024);
    CheckpointRecord expCheckpoint = new CheckpointRecord(new WALPointer(5738, 0, 0));
    FixCountRecord expDeltaPage1 = new FixCountRecord(grpId, expPageId2, 4);
    FixCountRecord expDeltaPage2 = new FixCountRecord(grpId, expPageId3, 4);
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, expPageSnapshot), new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(455, grpId), dummyPage(1024, 455), 1024)), new IgniteBiTuple<>(NULL_PTR, expCheckpoint), new IgniteBiTuple<>(NULL_PTR, new MetastoreDataRecord("key", new byte[0])), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage1), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, 98348, 4)), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage2));
    IgniteWalIteratorFactory mockedFactory = mock(IgniteWalIteratorFactory.class);
    when(mockedFactory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    List<WALRecord> holder = new ArrayList<>();
    ScannerHandler recordCaptor = (rec) -> holder.add(rec.get2());
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId1));
    groupAndPageIds.add(new T2<>(grpId, expPageId2));
    groupAndPageIds.add(new T2<>(grpId, expPageId3));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), mockedFactory).findAllRecordsFor(groupAndPageIds).forEach(recordCaptor);
    // then: Should be find only expected value.
    assertEquals(4, holder.size());
    assertEquals(expPageSnapshot, holder.get(0));
    assertEquals(expCheckpoint, holder.get(1));
    assertEquals(expDeltaPage1, holder.get(2));
    assertEquals(expDeltaPage2, holder.get(3));
}