Search in sources :

Example 26 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class IgniteWalRecoveryTest method testApplyDeltaRecords.

/**
 * @throws Exception if failed.
 */
@Test
public void testApplyDeltaRecords() throws Exception {
    IgniteEx ignite0 = (IgniteEx) startGrid("node0");
    ignite0.cluster().active(true);
    IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
    for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
    GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    db.waitForCheckpoint("test");
    db.enableCheckpoints(false).get();
    // Log something to know where to start.
    WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
    info("Replay marker: " + ptr);
    for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
    info("Done removes...");
    for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    Map<FullPageId, byte[]> rolledPages = new HashMap<>();
    int pageSize = sharedCtx.database().pageSize();
    ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
    buf.order(ByteOrder.nativeOrder());
    // Now check that deltas can be correctly applied.
    try (WALIterator it = sharedCtx.wal().replay(ptr)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof PageSnapshot) {
                PageSnapshot page = (PageSnapshot) rec;
                CacheGroupContext gctx = sharedCtx.cache().cacheGroup(page.groupId());
                int realPageSize = gctx == null ? pageSize : gctx.dataRegion().pageMemory().realPageSize(page.groupId());
                byte[] pageData = page.pageData();
                if (pageData.length < realPageSize) {
                    buf.clear();
                    buf.put(pageData);
                    buf.flip();
                    sharedCtx.kernalContext().compress().decompressPage(buf, realPageSize);
                    pageData = new byte[realPageSize];
                    buf.position(0);
                    buf.get(pageData);
                }
                rolledPages.put(page.fullPageId(), pageData);
            } else if (rec instanceof PageDeltaRecord) {
                PageDeltaRecord delta = (PageDeltaRecord) rec;
                FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
                byte[] pageData = rolledPages.get(fullId);
                if (pageData == null) {
                    pageData = new byte[pageSize];
                    rolledPages.put(fullId, pageData);
                }
                assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
                buf.clear();
                buf.put(pageData);
                buf.flip();
                delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
                buf.get(pageData);
            }
        }
    }
    info("Done apply...");
    PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
    ByteBuffer bufWal = ByteBuffer.allocateDirect(pageSize);
    for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
        FullPageId fullId = entry.getKey();
        ignite0.context().cache().context().database().checkpointReadLock();
        try {
            long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), IoStatisticsHolderNoOp.INSTANCE, true);
            try {
                long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
                try {
                    byte[] data = entry.getValue();
                    if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
                        // Skip tracking pages.
                        continue;
                    // compacted pages in case of compaction is used.
                    if (walPageCompression != null && PageIO.getPageIO(bufPtr) instanceof CompactablePageIO) {
                        CompactablePageIO pageIO = PageIO.getPageIO(bufPtr);
                        buf.clear();
                        bufWal.clear();
                        int realPageSize = data.length;
                        pageIO.compactPage(GridUnsafe.wrapPointer(bufPtr, realPageSize), buf, realPageSize);
                        pageIO.compactPage(ByteBuffer.wrap(data).order(ByteOrder.nativeOrder()), bufWal, realPageSize);
                        bufPtr = GridUnsafe.bufferAddress(buf);
                        data = new byte[bufWal.limit()];
                        bufWal.rewind();
                        bufWal.get(data);
                    }
                    for (int i = 0; i < data.length; i++) assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
                } finally {
                    pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
                }
            } finally {
                pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        } finally {
            ignite0.context().cache().context().database().checkpointReadUnlock();
        }
    }
    ignite0.close();
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) HashMap(java.util.HashMap) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) ByteBuffer(java.nio.ByteBuffer) CompactablePageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO) IgniteEx(org.apache.ignite.internal.IgniteEx) PageMemoryEx(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.Test) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest)

Example 27 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class PageMemoryTracker method applyWalRecord.

/**
 * Apply WAL record to local memory region.
 */
private void applyWalRecord(WALRecord record) throws IgniteCheckedException {
    memoryRegionLock.readLock().lock();
    try {
        if (!started)
            return;
        if (record instanceof MemoryRecoveryRecord && !emptyPds) {
            synchronized (pageAllocatorMux) {
                pages.clear();
                lastPageIdx = 0;
                freeSlotsCnt = maxPages;
                freeSlots.clear();
                stats.clear();
            }
        } else if (record instanceof PageSnapshot) {
            PageSnapshot snapshot = (PageSnapshot) record;
            int grpId = snapshot.fullPageId().groupId();
            long pageId = snapshot.fullPageId().pageId();
            FullPageId fullPageId = new FullPageId(pageId, grpId);
            DirectMemoryPage page = page(fullPageId);
            page.lock();
            try {
                GridUnsafe.copyHeapOffheap(snapshot.pageData(), GridUnsafe.BYTE_ARR_OFF, page.address(), pageSize);
                page.changeHistory().clear();
                page.changeHistory().add(record);
            } finally {
                page.unlock();
            }
        } else if (record instanceof PageDeltaRecord) {
            PageDeltaRecord deltaRecord = (PageDeltaRecord) record;
            int grpId = deltaRecord.groupId();
            long pageId = deltaRecord.pageId();
            FullPageId fullPageId = new FullPageId(pageId, grpId);
            DirectMemoryPage page = page(fullPageId);
            page.lock();
            try {
                deltaRecord.applyDelta(pageMemoryMock, page.address());
                page.changeHistory().add(record);
            } finally {
                page.unlock();
            }
        } else
            return;
        // Increment statistics.
        stats.computeIfAbsent(record.type(), r -> new AtomicInteger()).incrementAndGet();
    } finally {
        memoryRegionLock.readLock().unlock();
    }
}
Also used : T_DATA_REF_MVCC_LEAF(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.T_DATA_REF_MVCC_LEAF) DataRegion(org.apache.ignite.internal.processors.cache.persistence.DataRegion) TxLog(org.apache.ignite.internal.processors.cache.mvcc.txlog.TxLog) IgniteEx(org.apache.ignite.internal.IgniteEx) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) ByteBuffer(java.nio.ByteBuffer) UnsafeMemoryProvider(org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) CompactablePageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.CompactablePageIO) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) T_CACHE_ID_DATA_REF_MVCC_LEAF(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.T_CACHE_ID_DATA_REF_MVCC_LEAF) EncryptionSpi(org.apache.ignite.spi.encryption.EncryptionSpi) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) MetaStorage(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage) DirectMemoryProvider(org.apache.ignite.internal.mem.DirectMemoryProvider) Collectors(java.util.stream.Collectors) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) List(java.util.List) IgnitePageStoreManager(org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager) PageIdUtils(org.apache.ignite.internal.pagemem.PageIdUtils) CU(org.apache.ignite.internal.util.typedef.internal.CU) IgniteWriteAheadLogManager(org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) Mockito.mock(org.mockito.Mockito.mock) FileWriteAheadLogManager(org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) DataRegionMetricsImpl(org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) PageMemory(org.apache.ignite.internal.pagemem.PageMemory) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) DirectMemoryRegion(org.apache.ignite.internal.mem.DirectMemoryRegion) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) ConcurrentMap(java.util.concurrent.ConcurrentMap) GridKernalContext(org.apache.ignite.internal.GridKernalContext) HashSet(java.util.HashSet) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) IgnitePlugin(org.apache.ignite.plugin.IgnitePlugin) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) MvccUtils(org.apache.ignite.internal.processors.cache.mvcc.MvccUtils) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) CheckpointListener(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener) LinkedList(java.util.LinkedList) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) AbstractDataLeafIO(org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO) ReentrantLock(java.util.concurrent.locks.ReentrantLock) GridUnsafe(org.apache.ignite.internal.util.GridUnsafe) Mockito.when(org.mockito.Mockito.when) PageMemoryImpl(org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl) Mockito(org.mockito.Mockito) Lock(java.util.concurrent.locks.Lock) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) BitSet(java.util.BitSet) PluginContext(org.apache.ignite.plugin.PluginContext) DataRegionConfiguration(org.apache.ignite.configuration.DataRegionConfiguration) GridCacheProcessor(org.apache.ignite.internal.processors.cache.GridCacheProcessor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PageDeltaRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord) MemoryRecoveryRecord(org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId)

Example 28 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalScannerTest method shouldDumpToFileAndLogFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToFileAndLogFoundRecord() throws Exception {
    // given: File for dumping records and test logger for interception of records.
    File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
    long expPageId = 984;
    int grpId = 123;
    IgniteLogger log = mock(IgniteLogger.class);
    when(log.isInfoEnabled()).thenReturn(true);
    ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
    doNothing().when(log).info(valCapture.capture());
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    List<String> actualFileRecords = null;
    try {
        // when: Scanning WAL for searching expected page.
        buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log).andThen(printToFile(targetFile)));
        actualFileRecords = Files.readAllLines(targetFile.toPath());
    } finally {
        targetFile.delete();
    }
    actualFileRecords = actualFileRecords.stream().filter(it -> it.startsWith("Next WAL record ::")).collect(Collectors.toList());
    // then: Should be find only expected value from file.
    assertEquals(actualFileRecords.size(), 3);
    assertTrue(actualFileRecords.get(0), actualFileRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualFileRecords.get(1), actualFileRecords.get(1).contains("CheckpointRecord ["));
    assertTrue(actualFileRecords.get(2), actualFileRecords.get(2).contains("FixCountRecord ["));
    // then: Should be find only expected value from log.
    List<String> actualLogRecords = valCapture.getAllValues();
    assertEquals(actualLogRecords.size(), 1);
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("CheckpointRecord ["));
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("FixCountRecord ["));
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) FixCountRecord(org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) IgniteLogger(org.apache.ignite.IgniteLogger) ScannerHandlers.printToFile(org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToFile) File(java.io.File) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) T2(org.apache.ignite.internal.util.typedef.T2) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 29 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class WalScannerTest method shouldFindCorrectRecords.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldFindCorrectRecords() throws Exception {
    // given: Iterator with random value and value which should be find by scanner.
    long expPageId = 984;
    int grpId = 123;
    PageSnapshot expPageSnapshot = new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024);
    CheckpointRecord expCheckpoint = new CheckpointRecord(new WALPointer(5738, 0, 0));
    FixCountRecord expDeltaPage = new FixCountRecord(grpId, expPageId, 4);
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, expPageSnapshot), new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(455, grpId), dummyPage(4096, 455), 1024)), new IgniteBiTuple<>(NULL_PTR, expCheckpoint), new IgniteBiTuple<>(NULL_PTR, new MetastoreDataRecord("key", new byte[0])), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, 98348, 4)));
    IgniteWalIteratorFactory mockedFactory = mock(IgniteWalIteratorFactory.class);
    when(mockedFactory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    // Test scanner handler for holding found value instead of printing its.
    List<WALRecord> holder = new ArrayList<>();
    ScannerHandler recordCaptor = (rec) -> holder.add(rec.get2());
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), mockedFactory).findAllRecordsFor(groupAndPageIds).forEach(recordCaptor);
    // then: Should be find only expected value.
    assertEquals(holder.size(), 3);
    assertEquals(expPageSnapshot, holder.get(0));
    assertEquals(expCheckpoint, holder.get(1));
    assertEquals(expDeltaPage, holder.get(2));
}
Also used : IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) Arrays(java.util.Arrays) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) IgniteWalIteratorFactory(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory) U(org.apache.ignite.internal.util.typedef.internal.U) IgniteLogger(org.apache.ignite.IgniteLogger) FixCountRecord(org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord) IteratorParametersBuilder.withIteratorParameters(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder.withIteratorParameters) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ArgumentCaptor(org.mockito.ArgumentCaptor) ScannerHandlers.printToFile(org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToFile) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) Files(java.nio.file.Files) WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) Set(java.util.Set) GridUnsafe(org.apache.ignite.internal.util.GridUnsafe) Test(org.junit.Test) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) Mockito.doNothing(org.mockito.Mockito.doNothing) Mockito.when(org.mockito.Mockito.when) NULL_PTR(org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointStatus.NULL_PTR) Collectors(java.util.stream.Collectors) File(java.io.File) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) T2(org.apache.ignite.internal.util.typedef.T2) IgniteBiTuple(org.apache.ignite.lang.IgniteBiTuple) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) List(java.util.List) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) Paths(java.nio.file.Paths) DummyPageIO(org.apache.ignite.internal.processors.cache.persistence.DummyPageIO) TestCase.assertTrue(junit.framework.TestCase.assertTrue) WalScanner.buildWalScanner(org.apache.ignite.internal.processors.cache.persistence.wal.scanner.WalScanner.buildWalScanner) ScannerHandlers.printToLog(org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToLog) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.mock(org.mockito.Mockito.mock) PartitionMetaStateRecord(org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord) IteratorParametersBuilder(org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder) ArrayList(java.util.ArrayList) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) FixCountRecord(org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord) WALIterator(org.apache.ignite.internal.pagemem.wal.WALIterator) MetastoreDataRecord(org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) T2(org.apache.ignite.internal.util.typedef.T2) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 30 with PageSnapshot

use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testPageWalEntries.

/**
 * @throws Exception if failed.
 */
@Test
public void testPageWalEntries() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
    int cacheId = sharedCtx.cache().cache(CACHE_NAME).context().cacheId();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    PageMemory pageMem = sharedCtx.database().dataRegion(null).pageMemory();
    IgniteWriteAheadLogManager wal = sharedCtx.wal();
    db.enableCheckpoints(false).get();
    int pageCnt = 100;
    List<FullPageId> pageIds = new ArrayList<>();
    for (int i = 0; i < pageCnt; i++) {
        db.checkpointReadLock();
        try {
            pageIds.add(new FullPageId(pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX), cacheId));
        } finally {
            db.checkpointReadUnlock();
        }
    }
    UUID cpId = UUID.randomUUID();
    WALPointer start = wal.log(new CheckpointRecord(cpId, null));
    wal.flush(start, false);
    ig.context().cache().context().database().checkpointReadLock();
    try {
        for (FullPageId pageId : pageIds) writePageData(pageId, pageMem);
    } finally {
        ig.context().cache().context().database().checkpointReadUnlock();
    }
    // Data will not be written to the page store.
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().active(true);
    sharedCtx = ig.context().cache().context();
    db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    wal = sharedCtx.wal();
    db.enableCheckpoints(false);
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
        assert tup.get2() instanceof CheckpointRecord : tup.get2();
        assertEquals(start, tup.get1());
        CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
        assertEquals(cpId, cpRec.checkpointId());
        assertNull(cpRec.checkpointMark());
        assertFalse(cpRec.end());
        int idx = 0;
        while (idx < pageIds.size()) {
            tup = it.next();
            assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
            PageSnapshot snap = (PageSnapshot) tup.get2();
            // there are extra tracking pages, skip them
            long trackingPageFor = TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize());
            if (trackingPageFor == snap.fullPageId().pageId()) {
                tup = it.next();
                assertTrue(tup.get2() instanceof PageSnapshot);
                snap = (PageSnapshot) tup.get2();
            }
            assertEquals(pageIds.get(idx), snap.fullPageId());
            idx++;
        }
    }
}
Also used : WALRecord(org.apache.ignite.internal.pagemem.wal.record.WALRecord) IgniteWriteAheadLogManager(org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager) GridCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager) PageMemory(org.apache.ignite.internal.pagemem.PageMemory) ArrayList(java.util.ArrayList) CheckpointRecord(org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord) IgniteEx(org.apache.ignite.internal.IgniteEx) CacheObject(org.apache.ignite.internal.processors.cache.CacheObject) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) UUID(java.util.UUID) WALPointer(org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer) FullPageId(org.apache.ignite.internal.pagemem.FullPageId) PageSnapshot(org.apache.ignite.internal.pagemem.wal.record.PageSnapshot) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Test(org.junit.Test)

Aggregations

PageSnapshot (org.apache.ignite.internal.pagemem.wal.record.PageSnapshot)31 FullPageId (org.apache.ignite.internal.pagemem.FullPageId)23 WALPointer (org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer)19 CheckpointRecord (org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord)18 WALRecord (org.apache.ignite.internal.pagemem.wal.record.WALRecord)17 Test (org.junit.Test)13 File (java.io.File)12 WALIterator (org.apache.ignite.internal.pagemem.wal.WALIterator)12 ByteBuffer (java.nio.ByteBuffer)11 IgniteEx (org.apache.ignite.internal.IgniteEx)10 T2 (org.apache.ignite.internal.util.typedef.T2)10 ArrayList (java.util.ArrayList)9 MetastoreDataRecord (org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord)9 HashSet (java.util.HashSet)8 DataRecord (org.apache.ignite.internal.pagemem.wal.record.DataRecord)8 MemoryRecoveryRecord (org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord)8 HashMap (java.util.HashMap)7 UUID (java.util.UUID)7 FixCountRecord (org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord)6 Map (java.util.Map)5