use of org.apache.ignite.internal.pagemem.wal.record.WALRecord in project ignite by apache.
the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method verifyReads.
/**
* @param res Result map to verify.
* @param mem Memory.
*/
private void verifyReads(Map<FullPageId, Integer> res, PageMemory mem, WALPointer start, IgniteWriteAheadLogManager wal) throws Exception {
Map<FullPageId, byte[]> replay = new HashMap<>();
try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
while (it.hasNext()) {
tup = it.next();
WALRecord rec = tup.get2();
if (rec instanceof CheckpointRecord) {
CheckpointRecord end = (CheckpointRecord) rec;
// Found the finish mark.
if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
break;
} else if (rec instanceof PageSnapshot) {
PageSnapshot page = (PageSnapshot) rec;
replay.put(page.fullPageId(), page.pageData());
}
}
}
// Check read-through from the file store.
for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
FullPageId fullId = entry.getKey();
int state = entry.getValue();
if (state == -1) {
info("Page was never written: " + fullId);
continue;
}
byte[] walData = replay.get(fullId);
assertNotNull("Missing WAL record for a written page: " + fullId, walData);
long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
try {
long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
try {
for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
int expState = state & 0xFF;
int pageState = PageUtils.getByte(pageAddr, i) & 0xFF;
int walState = walData[i] & 0xFF;
if (expState != pageState)
assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']', expState, pageState);
if (expState != walState)
assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']', expState, walState);
}
} finally {
mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
}
} finally {
mem.releasePage(fullId.groupId(), fullId.pageId(), page);
}
}
}
use of org.apache.ignite.internal.pagemem.wal.record.WALRecord in project ignite by apache.
the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testDataWalEntries.
/**
* @throws Exception if failed.
*/
public void testDataWalEntries() throws Exception {
IgniteEx ig = startGrid(0);
ig.active(true);
GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(cacheName).context();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
IgniteWriteAheadLogManager wal = sharedCtx.wal();
assertTrue(wal.isAlwaysWriteFullPages());
db.enableCheckpoints(false).get();
final int cnt = 10;
List<DataEntry> entries = new ArrayList<>(cnt);
for (int i = 0; i < cnt; i++) {
GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
KeyCacheObject key = cctx.toCacheKeyObject(i);
CacheObject val = null;
if (op != GridCacheOperation.DELETE)
val = cctx.toCacheObject("value-" + i);
entries.add(new DataEntry(cctx.cacheId(), key, val, op, null, cctx.versions().next(), 0L, cctx.affinity().partition(i), i));
}
UUID cpId = UUID.randomUUID();
WALPointer start = wal.log(new CheckpointRecord(cpId, null));
wal.fsync(start);
for (DataEntry entry : entries) wal.log(new DataRecord(entry));
// Data will not be written to the page store.
stopAllGrids();
ig = startGrid(0);
ig.active(true);
sharedCtx = ig.context().cache().context();
cctx = sharedCtx.cache().cache(cacheName).context();
db = (GridCacheDatabaseSharedManager) sharedCtx.database();
wal = sharedCtx.wal();
db.enableCheckpoints(false).get();
try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
assert cpRecordTup.get2() instanceof CheckpointRecord;
assertEquals(start, cpRecordTup.get1());
CheckpointRecord cpRec = (CheckpointRecord) cpRecordTup.get2();
assertEquals(cpId, cpRec.checkpointId());
assertNull(cpRec.checkpointMark());
assertFalse(cpRec.end());
int idx = 0;
CacheObjectContext coctx = cctx.cacheObjectContext();
while (idx < entries.size()) {
IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
assert dataRecTup.get2() instanceof DataRecord;
DataRecord dataRec = (DataRecord) dataRecTup.get2();
DataEntry entry = entries.get(idx);
assertEquals(1, dataRec.writeEntries().size());
DataEntry readEntry = dataRec.writeEntries().get(0);
assertEquals(entry.cacheId(), readEntry.cacheId());
assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
assertEquals(entry.op(), readEntry.op());
if (entry.op() == GridCacheOperation.UPDATE)
assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
else
assertNull(entry.value());
assertEquals(entry.writeVersion(), readEntry.writeVersion());
assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
idx++;
}
}
}
use of org.apache.ignite.internal.pagemem.wal.record.WALRecord in project ignite by apache.
the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testPageWalEntries.
/**
* @throws Exception if failed.
*/
public void testPageWalEntries() throws Exception {
IgniteEx ig = startGrid(0);
ig.active(true);
GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
PageMemory pageMem = sharedCtx.database().dataRegion(null).pageMemory();
IgniteWriteAheadLogManager wal = sharedCtx.wal();
db.enableCheckpoints(false).get();
int pageCnt = 100;
List<FullPageId> pageIds = new ArrayList<>();
for (int i = 0; i < pageCnt; i++) {
db.checkpointReadLock();
try {
pageIds.add(new FullPageId(pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX), cacheId));
} finally {
db.checkpointReadUnlock();
}
}
UUID cpId = UUID.randomUUID();
WALPointer start = wal.log(new CheckpointRecord(cpId, null));
wal.fsync(start);
ig.context().cache().context().database().checkpointReadLock();
try {
for (FullPageId pageId : pageIds) writePageData(pageId, pageMem);
} finally {
ig.context().cache().context().database().checkpointReadUnlock();
}
// Data will not be written to the page store.
stopAllGrids();
ig = startGrid(0);
ig.active(true);
sharedCtx = ig.context().cache().context();
db = (GridCacheDatabaseSharedManager) sharedCtx.database();
wal = sharedCtx.wal();
db.enableCheckpoints(false);
try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assert tup.get2() instanceof CheckpointRecord : tup.get2();
assertEquals(start, tup.get1());
CheckpointRecord cpRec = (CheckpointRecord) tup.get2();
assertEquals(cpId, cpRec.checkpointId());
assertNull(cpRec.checkpointMark());
assertFalse(cpRec.end());
int idx = 0;
while (idx < pageIds.size()) {
tup = it.next();
assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
PageSnapshot snap = (PageSnapshot) tup.get2();
// there are extra tracking pages, skip them
if (TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize()) == snap.fullPageId().pageId()) {
tup = it.next();
assertTrue(tup.get2() instanceof PageSnapshot);
snap = (PageSnapshot) tup.get2();
}
assertEquals(pageIds.get(idx), snap.fullPageId());
idx++;
}
}
}
use of org.apache.ignite.internal.pagemem.wal.record.WALRecord in project ignite by apache.
the class IgniteWalRecoveryTest method testTxRecordsConsistency.
/**
* Test that all DataRecord WAL records are within transaction boundaries - PREPARED and COMMITTED markers.
*
* @throws Exception If any fail.
*/
public void testTxRecordsConsistency() throws Exception {
System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
IgniteEx ignite = (IgniteEx) startGrids(3);
ignite.active(true);
try {
final String cacheName = "transactional";
CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName).setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).setAffinity(new RendezvousAffinityFunction(false, 32)).setCacheMode(CacheMode.PARTITIONED).setRebalanceMode(CacheRebalanceMode.SYNC).setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC).setBackups(0);
ignite.createCache(cacheConfiguration);
IgniteCache<Object, Object> cache = ignite.cache(cacheName);
GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
db.waitForCheckpoint("test");
db.enableCheckpoints(false).get();
// Log something to know where to start.
WALPointer startPtr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
final int transactions = 100;
final int operationsPerTransaction = 40;
Random random = new Random();
for (int t = 1; t <= transactions; t++) {
Transaction tx = ignite.transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
for (int op = 0; op < operationsPerTransaction; op++) {
int key = random.nextInt(1000) + 1;
Object value;
if (random.nextBoolean())
value = randomString(random) + key;
else
value = new BigObject(key);
cache.put(key, value);
}
if (random.nextBoolean()) {
tx.commit();
} else {
tx.rollback();
}
if (t % 50 == 0)
log.info("Finished transaction " + t);
}
Set<GridCacheVersion> activeTransactions = new HashSet<>();
// Check that all DataRecords are within PREPARED and COMMITTED tx records.
try (WALIterator it = sharedCtx.wal().replay(startPtr)) {
while (it.hasNext()) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
WALRecord rec = tup.get2();
if (rec instanceof TxRecord) {
TxRecord txRecord = (TxRecord) rec;
GridCacheVersion txId = txRecord.nearXidVersion();
switch(txRecord.state()) {
case PREPARED:
assert !activeTransactions.contains(txId) : "Transaction is already present " + txRecord;
activeTransactions.add(txId);
break;
case COMMITTED:
assert activeTransactions.contains(txId) : "No PREPARE marker for transaction " + txRecord;
activeTransactions.remove(txId);
break;
case ROLLED_BACK:
activeTransactions.remove(txId);
break;
default:
throw new IllegalStateException("Unknown Tx state of record " + txRecord);
}
} else if (rec instanceof DataRecord) {
DataRecord dataRecord = (DataRecord) rec;
for (DataEntry entry : dataRecord.writeEntries()) {
GridCacheVersion txId = entry.nearXidVersion();
assert activeTransactions.contains(txId) : "No transaction for entry " + entry;
}
}
}
}
} finally {
System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
stopAllGrids();
}
}
use of org.apache.ignite.internal.pagemem.wal.record.WALRecord in project ignite by apache.
the class IgniteWalRecoveryTest method testApplyDeltaRecords.
/**
* @throws Exception if failed.
*/
public void testApplyDeltaRecords() throws Exception {
try {
IgniteEx ignite0 = (IgniteEx) startGrid("node0");
ignite0.active(true);
IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
db.waitForCheckpoint("test");
db.enableCheckpoints(false).get();
// Log something to know where to start.
WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
info("Replay marker: " + ptr);
for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
info("Done puts...");
for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
info("Done removes...");
for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
info("Done puts...");
Map<FullPageId, byte[]> rolledPages = new HashMap<>();
int pageSize = sharedCtx.database().pageSize();
ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
// Now check that deltas can be correctly applied.
try (WALIterator it = sharedCtx.wal().replay(ptr)) {
while (it.hasNext()) {
IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
WALRecord rec = tup.get2();
if (rec instanceof PageSnapshot) {
PageSnapshot page = (PageSnapshot) rec;
rolledPages.put(page.fullPageId(), page.pageData());
} else if (rec instanceof PageDeltaRecord) {
PageDeltaRecord delta = (PageDeltaRecord) rec;
FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
byte[] pageData = rolledPages.get(fullId);
if (pageData == null) {
pageData = new byte[pageSize];
rolledPages.put(fullId, pageData);
}
assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
buf.order(ByteOrder.nativeOrder());
buf.position(0);
buf.put(pageData);
buf.position(0);
delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
buf.position(0);
buf.get(pageData);
}
}
}
info("Done apply...");
PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
FullPageId fullId = entry.getKey();
ignite0.context().cache().context().database().checkpointReadLock();
try {
long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), true);
try {
long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
try {
byte[] data = entry.getValue();
for (int i = 0; i < data.length; i++) {
if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
// Skip tracking pages.
continue;
assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
}
} finally {
pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
}
} finally {
pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
}
} finally {
ignite0.context().cache().context().database().checkpointReadUnlock();
}
}
ignite0.close();
} finally {
stopAllGrids();
}
}
Aggregations