use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class IgniteWalRecoveryTest method testApplyDeltaRecords.
/**
* @throws Exception if failed.
*/
@Test
public void testApplyDeltaRecords() throws Exception {
IgniteEx ignite0 = (IgniteEx) startGrid("node0");
ignite0.cluster().active(true);
IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
db.waitForCheckpoint("test");
db.enableCheckpoints(false).get();
// Log something to know where to start.
WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
info("Replay marker: " + ptr);
for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
info("Done puts...");
for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
info("Done removes...");
for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
info("Done puts...");
Map<FullPageId, byte[]> rolledPages = new HashMap<>();
int pageSize = sharedCtx.database().pageSize();
ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
buf.order(ByteOrder.nativeOrder());
// Now check that deltas can be correctly applied.
try (WALIterator it = sharedCtx.wal().replay(ptr)) {
while (it.hasNext()) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
WALRecord rec = tup.get2();
if (rec instanceof PageSnapshot) {
PageSnapshot page = (PageSnapshot) rec;
CacheGroupContext gctx = sharedCtx.cache().cacheGroup(page.groupId());
int realPageSize = gctx == null ? pageSize : gctx.dataRegion().pageMemory().realPageSize(page.groupId());
byte[] pageData = page.pageData();
if (pageData.length < realPageSize) {
buf.clear();
buf.put(pageData);
buf.flip();
sharedCtx.kernalContext().compress().decompressPage(buf, realPageSize);
pageData = new byte[realPageSize];
buf.position(0);
buf.get(pageData);
}
rolledPages.put(page.fullPageId(), pageData);
} else if (rec instanceof PageDeltaRecord) {
PageDeltaRecord delta = (PageDeltaRecord) rec;
FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
byte[] pageData = rolledPages.get(fullId);
if (pageData == null) {
pageData = new byte[pageSize];
rolledPages.put(fullId, pageData);
}
assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
buf.clear();
buf.put(pageData);
buf.flip();
delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
buf.get(pageData);
}
}
}
info("Done apply...");
PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
ByteBuffer bufWal = ByteBuffer.allocateDirect(pageSize);
for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
FullPageId fullId = entry.getKey();
ignite0.context().cache().context().database().checkpointReadLock();
try {
long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), IoStatisticsHolderNoOp.INSTANCE, true);
try {
long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
try {
byte[] data = entry.getValue();
if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
// Skip tracking pages.
continue;
// compacted pages in case of compaction is used.
if (walPageCompression != null && PageIO.getPageIO(bufPtr) instanceof CompactablePageIO) {
CompactablePageIO pageIO = PageIO.getPageIO(bufPtr);
buf.clear();
bufWal.clear();
int realPageSize = data.length;
pageIO.compactPage(GridUnsafe.wrapPointer(bufPtr, realPageSize), buf, realPageSize);
pageIO.compactPage(ByteBuffer.wrap(data).order(ByteOrder.nativeOrder()), bufWal, realPageSize);
bufPtr = GridUnsafe.bufferAddress(buf);
data = new byte[bufWal.limit()];
bufWal.rewind();
bufWal.get(data);
}
for (int i = 0; i < data.length; i++) assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
} finally {
pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
}
} finally {
pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
}
} finally {
ignite0.context().cache().context().database().checkpointReadUnlock();
}
}
ignite0.close();
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class PageMemoryTracker method applyWalRecord.
/**
* Apply WAL record to local memory region.
*/
private void applyWalRecord(WALRecord record) throws IgniteCheckedException {
memoryRegionLock.readLock().lock();
try {
if (!started)
return;
if (record instanceof MemoryRecoveryRecord && !emptyPds) {
synchronized (pageAllocatorMux) {
pages.clear();
lastPageIdx = 0;
freeSlotsCnt = maxPages;
freeSlots.clear();
stats.clear();
}
} else if (record instanceof PageSnapshot) {
PageSnapshot snapshot = (PageSnapshot) record;
int grpId = snapshot.fullPageId().groupId();
long pageId = snapshot.fullPageId().pageId();
FullPageId fullPageId = new FullPageId(pageId, grpId);
DirectMemoryPage page = page(fullPageId);
page.lock();
try {
GridUnsafe.copyHeapOffheap(snapshot.pageData(), GridUnsafe.BYTE_ARR_OFF, page.address(), pageSize);
page.changeHistory().clear();
page.changeHistory().add(record);
} finally {
page.unlock();
}
} else if (record instanceof PageDeltaRecord) {
PageDeltaRecord deltaRecord = (PageDeltaRecord) record;
int grpId = deltaRecord.groupId();
long pageId = deltaRecord.pageId();
FullPageId fullPageId = new FullPageId(pageId, grpId);
DirectMemoryPage page = page(fullPageId);
page.lock();
try {
deltaRecord.applyDelta(pageMemoryMock, page.address());
page.changeHistory().add(record);
} finally {
page.unlock();
}
} else
return;
// Increment statistics.
stats.computeIfAbsent(record.type(), r -> new AtomicInteger()).incrementAndGet();
} finally {
memoryRegionLock.readLock().unlock();
}
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class WalScannerTest method shouldDumpToFileAndLogFoundRecord.
/**
* @throws Exception If failed.
*/
@Test
public void shouldDumpToFileAndLogFoundRecord() throws Exception {
// given: File for dumping records and test logger for interception of records.
File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
long expPageId = 984;
int grpId = 123;
IgniteLogger log = mock(IgniteLogger.class);
when(log.isInfoEnabled()).thenReturn(true);
ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
doNothing().when(log).info(valCapture.capture());
WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
groupAndPageIds.add(new T2<>(grpId, expPageId));
List<String> actualFileRecords = null;
try {
// when: Scanning WAL for searching expected page.
buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log).andThen(printToFile(targetFile)));
actualFileRecords = Files.readAllLines(targetFile.toPath());
} finally {
targetFile.delete();
}
actualFileRecords = actualFileRecords.stream().filter(it -> it.startsWith("Next WAL record ::")).collect(Collectors.toList());
// then: Should be find only expected value from file.
assertEquals(actualFileRecords.size(), 3);
assertTrue(actualFileRecords.get(0), actualFileRecords.get(0).contains("PageSnapshot ["));
assertTrue(actualFileRecords.get(1), actualFileRecords.get(1).contains("CheckpointRecord ["));
assertTrue(actualFileRecords.get(2), actualFileRecords.get(2).contains("FixCountRecord ["));
// then: Should be find only expected value from log.
List<String> actualLogRecords = valCapture.getAllValues();
assertEquals(actualLogRecords.size(), 1);
assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("PageSnapshot ["));
assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("CheckpointRecord ["));
assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("FixCountRecord ["));
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class WalScannerTest method shouldFindCorrectRecords.
/**
* @throws Exception If failed.
*/
@Test
public void shouldFindCorrectRecords() throws Exception {
// given: Iterator with random value and value which should be find by scanner.
long expPageId = 984;
int grpId = 123;
PageSnapshot expPageSnapshot = new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024);
CheckpointRecord expCheckpoint = new CheckpointRecord(new WALPointer(5738, 0, 0));
FixCountRecord expDeltaPage = new FixCountRecord(grpId, expPageId, 4);
WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, expPageSnapshot), new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(455, grpId), dummyPage(4096, 455), 1024)), new IgniteBiTuple<>(NULL_PTR, expCheckpoint), new IgniteBiTuple<>(NULL_PTR, new MetastoreDataRecord("key", new byte[0])), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, 98348, 4)));
IgniteWalIteratorFactory mockedFactory = mock(IgniteWalIteratorFactory.class);
when(mockedFactory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
// Test scanner handler for holding found value instead of printing its.
List<WALRecord> holder = new ArrayList<>();
ScannerHandler recordCaptor = (rec) -> holder.add(rec.get2());
Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
groupAndPageIds.add(new T2<>(grpId, expPageId));
// when: Scanning WAL for searching expected page.
buildWalScanner(withIteratorParameters(), mockedFactory).findAllRecordsFor(groupAndPageIds).forEach(recordCaptor);
// then: Should be find only expected value.
assertEquals(holder.size(), 3);
assertEquals(expPageSnapshot, holder.get(0));
assertEquals(expCheckpoint, holder.get(1));
assertEquals(expDeltaPage, holder.get(2));
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testPageWalEntries.
/**
* @throws Exception if failed.
*/
@Test
public void testPageWalEntries() throws Exception {
IgniteEx ig = startGrid(0);
ig.cluster().active(true);
GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
int cacheId = sharedCtx.cache().cache(CACHE_NAME).context().cacheId();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
PageMemory pageMem = sharedCtx.database().dataRegion(null).pageMemory();
IgniteWriteAheadLogManager wal = sharedCtx.wal();
db.enableCheckpoints(false).get();
int pageCnt = 100;
List<FullPageId> pageIds = new ArrayList<>();
for (int i = 0; i < pageCnt; i++) {
db.checkpointReadLock();
try {
pageIds.add(new FullPageId(pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX), cacheId));
} finally {
db.checkpointReadUnlock();
}
}
UUID cpId = UUID.randomUUID();
WALPointer start = wal.log(new CheckpointRecord(cpId, null));
wal.flush(start, false);
ig.context().cache().context().database().checkpointReadLock();
try {
for (FullPageId pageId : pageIds) writePageData(pageId, pageMem);
} finally {
ig.context().cache().context().database().checkpointReadUnlock();
}
// Data will not be written to the page store.
stopAllGrids();
ig = startGrid(0);
ig.cluster().active(true);
sharedCtx = ig.context().cache().context();
db = (GridCacheDatabaseSharedManager) sharedCtx.database();
wal = sharedCtx.wal();
db.enableCheckpoints(false);
try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assert tup.get2() instanceof CheckpointRecord : tup.get2();
assertEquals(start, tup.get1());
CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
assertEquals(cpId, cpRec.checkpointId());
assertNull(cpRec.checkpointMark());
assertFalse(cpRec.end());
int idx = 0;
while (idx < pageIds.size()) {
tup = it.next();
assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
PageSnapshot snap = (PageSnapshot) tup.get2();
// there are extra tracking pages, skip them
long trackingPageFor = TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize());
if (trackingPageFor == snap.fullPageId().pageId()) {
tup = it.next();
assertTrue(tup.get2() instanceof PageSnapshot);
snap = (PageSnapshot) tup.get2();
}
assertEquals(pageIds.get(idx), snap.fullPageId());
idx++;
}
}
}
Aggregations