Examples with WALPointer org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer used on opensource projects

Example 46 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgnitePdsRecoveryAfterFileCorruptionTest method testPageRecoveryAfterFileCorruption.

/**
 * @throws Exception if failed.
 */
@Test
public void testPageRecoveryAfterFileCorruption() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
    // Put for create data store and init meta page.
    cache.put(1, 1);
    GridCacheSharedContext sharedCtx = ig.context().cache().context();
    GridCacheDatabaseSharedManager psMgr = (GridCacheDatabaseSharedManager) sharedCtx.database();
    FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
    U.sleep(1_000);
    // Disable integrated checkpoint thread.
    psMgr.enableCheckpoints(false).get();
    PageMemory mem = sharedCtx.database().dataRegion(policyName).pageMemory();
    DummyPageIO pageIO = new DummyPageIO();
    int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
    int pagesCnt = getTotalPagesToTest();
    FullPageId[] pages = new FullPageId[pagesCnt];
    // Get lock to prevent assertion. A new page should be allocated under checkpoint lock.
    psMgr.checkpointReadLock();
    try {
        for (int i = 0; i < pagesCnt; i++) {
            pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
            initPage(mem, pageIO, pages[i]);
        }
        generateWal((PageMemoryImpl) mem, sharedCtx.pageStore(), sharedCtx.wal(), cacheId, pages);
    } finally {
        psMgr.checkpointReadUnlock();
    }
    eraseDataFromDisk(pageStore, cacheId, pages[0]);
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    checkRestore(ig, pages);
    // It is necessary to clear the current WAL history to make sure that the restored pages have been saved.
    GridCacheSharedContext<Object, Object> cctx = ig.context().cache().context();
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) cctx.database();
    FileWriteAheadLogManager wal = (FileWriteAheadLogManager) cctx.wal();
    // Force checkpoint.
    dbMgr.enableCheckpoints(true).get(getTestTimeout());
    dbMgr.checkpointReadLock();
    try {
        WALPointer lastWalPtr = dbMgr.checkpointHistory().lastCheckpoint().checkpointMark();
        // Move current WAL segment into the archive.
        wal.log(new CheckpointRecord(null), RolloverType.NEXT_SEGMENT);
        assertTrue(waitForCondition(() -> wal.lastArchivedSegment() >= lastWalPtr.index(), getTestTimeout()));
        wal.truncate(lastWalPtr);
        dbMgr.onWalTruncated(lastWalPtr);
    } finally {
        dbMgr.checkpointReadUnlock();
    }
    stopAllGrids();
    ig = startGrid(0);
    ig.cluster().state(ClusterState.ACTIVE);
    checkRestore(ig, pages);
}

Example 47 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgnitePdsSporadicDataRecordsOnBackupTest method findSporadicDataRecords.

/**
 * Returns a number of {@link DataRecord} that do not have a xid version,
 * in other words, number of records that were created by GridCacheUtils#createBackupPostProcessingClosure().
 *
 * @throws IgniteCheckedException If failed.
 */
private long findSporadicDataRecords(String nodeFolderName) throws IgniteCheckedException {
    File dbDir = new File(U.defaultWorkDirectory(), "db");
    File commonWalDir = new File(dbDir, "wal");
    File walDir = new File(commonWalDir, nodeFolderName);
    File walArchiveDir = new File(new File(commonWalDir, "archive"), nodeFolderName);
    assertTrue(walDir.exists());
    assertTrue(walArchiveDir.exists());
    IteratorParametersBuilder params = new IteratorParametersBuilder();
    params.bufferSize(1024 * 1024);
    params.filesOrDirs(walDir, walArchiveDir);
    params.filter((type, pointer) -> type == WALRecord.RecordType.DATA_RECORD_V2);
    int cacheId = CU.cacheId(TX_CACHE_NAME);
    long createOpCnt = 0;
    try (WALIterator itr = new IgniteWalIteratorFactory().iterator(params)) {
        while (itr.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> walEntry = itr.next();
            assertTrue(walEntry.get2() instanceof DataRecord);
            DataRecord rec = (DataRecord) walEntry.get2();
            Predicate<DataEntry> filter = e -> e.cacheId() == cacheId && GridCacheOperation.CREATE == e.op() && e.nearXidVersion() == null;
            for (int i = 0; i < rec.entryCount(); i++) {
                if (filter.test(rec.get(i)))
                    createOpCnt++;
            }
        }
    }
    return createOpCnt;
}

Example 48 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalScannerTest method shouldFindCorrectRecordsForMoreThanOnePages.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldFindCorrectRecordsForMoreThanOnePages() throws Exception {
    // given: Iterator with random value and value which should be find by scanner with several ids.
    long expPageId1 = 984;
    long expPageId2 = 9584;
    long expPageId3 = 98344;
    int grpId = 123;
    PageSnapshot expPageSnapshot = new PageSnapshot(new FullPageId(expPageId1, grpId), dummyPage(1024, expPageId1), 1024);
    CheckpointRecord expCheckpoint = new CheckpointRecord(new WALPointer(5738, 0, 0));
    FixCountRecord expDeltaPage1 = new FixCountRecord(grpId, expPageId2, 4);
    FixCountRecord expDeltaPage2 = new FixCountRecord(grpId, expPageId3, 4);
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, expPageSnapshot), new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(455, grpId), dummyPage(1024, 455), 1024)), new IgniteBiTuple<>(NULL_PTR, expCheckpoint), new IgniteBiTuple<>(NULL_PTR, new MetastoreDataRecord("key", new byte[0])), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage1), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, 98348, 4)), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage2));
    IgniteWalIteratorFactory mockedFactory = mock(IgniteWalIteratorFactory.class);
    when(mockedFactory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    List<WALRecord> holder = new ArrayList<>();
    ScannerHandler recordCaptor = (rec) -> holder.add(rec.get2());
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId1));
    groupAndPageIds.add(new T2<>(grpId, expPageId2));
    groupAndPageIds.add(new T2<>(grpId, expPageId3));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), mockedFactory).findAllRecordsFor(groupAndPageIds).forEach(recordCaptor);
    // then: Should be find only expected value.
    assertEquals(4, holder.size());
    assertEquals(expPageSnapshot, holder.get(0));
    assertEquals(expCheckpoint, holder.get(1));
    assertEquals(expDeltaPage1, holder.get(2));
    assertEquals(expDeltaPage2, holder.get(3));
}

Example 49 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalScannerTest method shouldDumpToLogFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToLogFoundRecord() throws Exception {
    // given: Test logger for interception of logging.
    long expPageId = 984;
    int grpId = 123;
    IgniteLogger log = mock(IgniteLogger.class);
    when(log.isInfoEnabled()).thenReturn(true);
    ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
    doNothing().when(log).info(valCapture.capture());
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log));
    // then: Should be find only expected value from log.
    List<String> actualRecords = valCapture.getAllValues();
    assertEquals(actualRecords.size(), 1);
    assertRecord(actualRecords.get(0), "PageSnapshot [", "PAGE_RECORD");
    assertRecord(actualRecords.get(0), "CheckpointRecord [", "CHECKPOINT_RECORD");
    assertRecord(actualRecords.get(0), "FixCountRecord [", "BTREE_FIX_COUNT");
}

Example 50 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalScannerTest method shouldDumpToFileFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToFileFoundRecord() throws Exception {
    // given: File for dumping records.
    File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
    long expectedPageId = 984;
    int grpId = 123;
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expectedPageId, grpId), dummyPage(1024, expectedPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expectedPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expectedPageId));
    List<String> actualRecords;
    try {
        // when: Scanning WAL for searching expected page.
        buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToFile(targetFile));
        actualRecords = Files.readAllLines(targetFile.toPath());
    } finally {
        targetFile.delete();
    }
    // then: Should be find only expected value from file. PageSnapshot string representation is 11 lines long.
    assertEquals(13, actualRecords.size());
    assertTrue(actualRecords.get(0), actualRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualRecords.get(11), actualRecords.get(11).contains("CheckpointRecord ["));
    assertTrue(actualRecords.get(12), actualRecords.get(12).contains("FixCountRecord ["));
}