use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class WalScannerTest method shouldDumpToLogFoundRecord.
/**
* @throws Exception If failed.
*/
@Test
public void shouldDumpToLogFoundRecord() throws Exception {
// given: Test logger for interception of logging.
long expPageId = 984;
int grpId = 123;
IgniteLogger log = mock(IgniteLogger.class);
when(log.isInfoEnabled()).thenReturn(true);
ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
doNothing().when(log).info(valCapture.capture());
WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
groupAndPageIds.add(new T2<>(grpId, expPageId));
// when: Scanning WAL for searching expected page.
buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log));
// then: Should be find only expected value from log.
List<String> actualRecords = valCapture.getAllValues();
assertEquals(actualRecords.size(), 1);
assertRecord(actualRecords.get(0), "PageSnapshot [", "PAGE_RECORD");
assertRecord(actualRecords.get(0), "CheckpointRecord [", "CHECKPOINT_RECORD");
assertRecord(actualRecords.get(0), "FixCountRecord [", "BTREE_FIX_COUNT");
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class WalScannerTest method shouldDumpToFileFoundRecord.
/**
* @throws Exception If failed.
*/
@Test
public void shouldDumpToFileFoundRecord() throws Exception {
// given: File for dumping records.
File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
long expectedPageId = 984;
int grpId = 123;
WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expectedPageId, grpId), dummyPage(1024, expectedPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expectedPageId, 4)));
IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
groupAndPageIds.add(new T2<>(grpId, expectedPageId));
List<String> actualRecords;
try {
// when: Scanning WAL for searching expected page.
buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToFile(targetFile));
actualRecords = Files.readAllLines(targetFile.toPath());
} finally {
targetFile.delete();
}
// then: Should be find only expected value from file. PageSnapshot string representation is 11 lines long.
assertEquals(13, actualRecords.size());
assertTrue(actualRecords.get(0), actualRecords.get(0).contains("PageSnapshot ["));
assertTrue(actualRecords.get(11), actualRecords.get(11).contains("CheckpointRecord ["));
assertTrue(actualRecords.get(12), actualRecords.get(12).contains("FixCountRecord ["));
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class RecordDataV1Serializer method size.
/**
* {@inheritDoc}
*/
@Override
public int size(WALRecord record) throws IgniteCheckedException {
switch(record.type()) {
case PAGE_RECORD:
assert record instanceof PageSnapshot;
PageSnapshot pageRec = (PageSnapshot) record;
return pageRec.pageData().length + 12;
case CHECKPOINT_RECORD:
CheckpointRecord cpRec = (CheckpointRecord) record;
assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer : "Invalid WAL record: " + cpRec;
int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
FileWALPointer walPtr = (FileWALPointer) cpRec.checkpointMark();
return 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
case META_PAGE_INIT:
return /*cache ID*/
4 + /*page ID*/
8 + /*ioType*/
2 + /*ioVer*/
2 + /*tree root*/
8 + /*reuse root*/
8;
case PARTITION_META_PAGE_UPDATE_COUNTERS:
return /*cache ID*/
4 + /*page ID*/
8 + /*upd cntr*/
8 + /*rmv id*/
8 + /*part size*/
4 + /*counters page id*/
8 + /*state*/
1 + /*allocatedIdxCandidate*/
4;
case MEMORY_RECOVERY:
return 8;
case PARTITION_DESTROY:
return /*cacheId*/
4 + /*partId*/
4;
case DATA_RECORD:
DataRecord dataRec = (DataRecord) record;
return 4 + dataSize(dataRec);
case METASTORE_DATA_RECORD:
MetastoreDataRecord metastoreDataRec = (MetastoreDataRecord) record;
return 4 + metastoreDataRec.key().getBytes().length + 4 + (metastoreDataRec.value() != null ? metastoreDataRec.value().length : 0);
case HEADER_RECORD:
return HEADER_RECORD_DATA_SIZE;
case DATA_PAGE_INSERT_RECORD:
DataPageInsertRecord diRec = (DataPageInsertRecord) record;
return 4 + 8 + 2 + diRec.payload().length;
case DATA_PAGE_UPDATE_RECORD:
DataPageUpdateRecord uRec = (DataPageUpdateRecord) record;
return 4 + 8 + 2 + 4 + uRec.payload().length;
case DATA_PAGE_INSERT_FRAGMENT_RECORD:
final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord) record;
return 4 + 8 + 8 + 4 + difRec.payloadSize();
case DATA_PAGE_REMOVE_RECORD:
return 4 + 8 + 1;
case DATA_PAGE_SET_FREE_LIST_PAGE:
return 4 + 8 + 8;
case INIT_NEW_PAGE_RECORD:
return 4 + 8 + 2 + 2 + 8;
case BTREE_META_PAGE_INIT_ROOT:
return 4 + 8 + 8;
case BTREE_META_PAGE_INIT_ROOT2:
return 4 + 8 + 8 + 2;
case BTREE_META_PAGE_ADD_ROOT:
return 4 + 8 + 8;
case BTREE_META_PAGE_CUT_ROOT:
return 4 + 8;
case BTREE_INIT_NEW_ROOT:
NewRootInitRecord<?> riRec = (NewRootInitRecord<?>) record;
return 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize();
case BTREE_PAGE_RECYCLE:
return 4 + 8 + 8;
case BTREE_PAGE_INSERT:
InsertRecord<?> inRec = (InsertRecord<?>) record;
return 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize();
case BTREE_FIX_LEFTMOST_CHILD:
return 4 + 8 + 8;
case BTREE_FIX_COUNT:
return 4 + 8 + 2;
case BTREE_PAGE_REPLACE:
ReplaceRecord<?> rRec = (ReplaceRecord<?>) record;
return 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize();
case BTREE_PAGE_REMOVE:
return 4 + 8 + 2 + 2;
case BTREE_PAGE_INNER_REPLACE:
return 4 + 8 + 2 + 8 + 2 + 8;
case BTREE_FORWARD_PAGE_SPLIT:
return 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2;
case BTREE_EXISTING_PAGE_SPLIT:
return 4 + 8 + 2 + 8;
case BTREE_PAGE_MERGE:
return 4 + 8 + 8 + 2 + 8 + 1;
case BTREE_FIX_REMOVE_ID:
return 4 + 8 + 8;
case PAGES_LIST_SET_NEXT:
return 4 + 8 + 8;
case PAGES_LIST_SET_PREVIOUS:
return 4 + 8 + 8;
case PAGES_LIST_INIT_NEW_PAGE:
return 4 + 8 + 4 + 4 + 8 + 8 + 8;
case PAGES_LIST_ADD_PAGE:
return 4 + 8 + 8;
case PAGES_LIST_REMOVE_PAGE:
return 4 + 8 + 8;
case TRACKING_PAGE_DELTA:
return 4 + 8 + 8 + 8 + 8;
case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
return 4 + 8 + 8 + 8;
case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
return 4 + 8 + 8;
case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
return 4 + 8 + 8;
case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
return 4 + 8 + 4;
case PART_META_UPDATE_STATE:
return /*cacheId*/
4 + /*partId*/
4 + /*State*/
1 + /*Update Counter*/
8;
case PAGE_LIST_META_RESET_COUNT_RECORD:
return /*cacheId*/
4 + /*pageId*/
8;
case SWITCH_SEGMENT_RECORD:
return 0;
case TX_RECORD:
return txRecordSerializer.size((TxRecord) record);
default:
throw new UnsupportedOperationException("Type: " + record.type());
}
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class GridCacheDatabaseSharedManager method restoreMemory.
/**
* @param status Checkpoint status.
* @param storeOnly If {@code True} restores Metastorage only.
*/
private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly, PageMemoryEx storePageMem) throws IgniteCheckedException {
assert !storeOnly || storePageMem != null;
if (log.isInfoEnabled())
log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked=" + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
boolean apply = status.needRestoreMemory();
if (apply) {
U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " + "finish checkpoint on node start.");
cctx.pageStore().beginRecover();
} else
cctx.wal().allowCompressionUntil(status.startPtr);
long start = U.currentTimeMillis();
int applied = 0;
WALPointer lastRead = null;
Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
try (WALIterator it = cctx.wal().replay(status.endPtr)) {
while (it.hasNextX()) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
WALRecord rec = tup.get2();
lastRead = tup.get1();
switch(rec.type()) {
case CHECKPOINT_RECORD:
CheckpointRecord cpRec = (CheckpointRecord) rec;
// We roll memory up until we find a checkpoint start record registered in the status.
if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() + ", pos=" + tup.get1() + ']');
apply = false;
} else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() + ", expCpId=" + status.cpStartId + ", pos=" + tup.get1() + ']');
break;
case PAGE_RECORD:
if (apply) {
PageSnapshot pageRec = (PageSnapshot) rec;
// Here we do not require tag check because we may be applying memory changes after
// several repetitive restarts and the same pages may have changed several times.
int grpId = pageRec.fullPageId().groupId();
if (storeOnly && grpId != METASTORAGE_CACHE_ID)
continue;
if (!ignoreGrps.contains(grpId)) {
long pageId = pageRec.fullPageId().pageId();
PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
long page = pageMem.acquirePage(grpId, pageId, true);
try {
long pageAddr = pageMem.writeLock(grpId, pageId, page);
try {
PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
} finally {
pageMem.writeUnlock(grpId, pageId, page, null, true, true);
}
} finally {
pageMem.releasePage(grpId, pageId, page);
}
applied++;
}
}
break;
case PARTITION_DESTROY:
PartitionDestroyRecord destroyRec = (PartitionDestroyRecord) rec;
final int gId = destroyRec.groupId();
if (storeOnly && gId != METASTORAGE_CACHE_ID)
continue;
if (!ignoreGrps.contains(gId)) {
final int pId = destroyRec.partitionId();
PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
pageMem.clearAsync((grpId, pageId) -> grpId == gId && PageIdUtils.partId(pageId) == pId, true).get();
}
break;
default:
if (apply && rec instanceof PageDeltaRecord) {
PageDeltaRecord r = (PageDeltaRecord) rec;
int grpId = r.groupId();
if (storeOnly && grpId != METASTORAGE_CACHE_ID)
continue;
if (!ignoreGrps.contains(grpId)) {
long pageId = r.pageId();
PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
// Here we do not require tag check because we may be applying memory changes after
// several repetitive restarts and the same pages may have changed several times.
long page = pageMem.acquirePage(grpId, pageId, true);
try {
long pageAddr = pageMem.writeLock(grpId, pageId, page);
try {
r.applyDelta(pageMem, pageAddr);
} finally {
pageMem.writeUnlock(grpId, pageId, page, null, true, true);
}
} finally {
pageMem.releasePage(grpId, pageId, page);
}
applied++;
}
}
}
}
}
if (storeOnly)
return null;
if (status.needRestoreMemory()) {
if (apply)
throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " + "on disk, but checkpoint record is missed in WAL) " + "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
log.info("Finished applying memory changes [changesApplied=" + applied + ", time=" + (U.currentTimeMillis() - start) + "ms]");
if (applied > 0)
finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
}
checkpointHist.loadHistory(cpDir);
return lastRead == null ? null : lastRead.next();
}
use of org.apache.ignite.internal.pagemem.wal.record.PageSnapshot in project ignite by apache.
the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method verifyReads.
/**
* @param res Result map to verify.
* @param mem Memory.
*/
private void verifyReads(Map<FullPageId, Integer> res, PageMemory mem, WALPointer start, IgniteWriteAheadLogManager wal) throws Exception {
Map<FullPageId, byte[]> replay = new HashMap<>();
try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
while (it.hasNext()) {
tup = it.next();
WALRecord rec = tup.get2();
if (rec instanceof CheckpointRecord) {
CheckpointRecord end = (CheckpointRecord) rec;
// Found the finish mark.
if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
break;
} else if (rec instanceof PageSnapshot) {
PageSnapshot page = (PageSnapshot) rec;
replay.put(page.fullPageId(), page.pageData());
}
}
}
// Check read-through from the file store.
for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
FullPageId fullId = entry.getKey();
int state = entry.getValue();
if (state == -1) {
info("Page was never written: " + fullId);
continue;
}
byte[] walData = replay.get(fullId);
assertNotNull("Missing WAL record for a written page: " + fullId, walData);
long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
try {
long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
try {
for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
int expState = state & 0xFF;
int pageState = PageUtils.getByte(pageAddr, i) & 0xFF;
int walState = walData[i] & 0xFF;
if (expState != pageState)
assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']', expState, pageState);
if (expState != walState)
assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']', expState, walState);
}
} finally {
mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
}
} finally {
mem.releasePage(fullId.groupId(), fullId.pageId(), page);
}
}
}
Aggregations