Examples with WALPointer org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer used on opensource projects

Example 91 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class CacheGroupKeyChangeTest method testWalArchiveCleanup.

/**
 * Ensures that unused key will be removed even if user cleaned wal archive folder manually.
 *
 * @throws Exception If failed.
 */
@Test
public void testWalArchiveCleanup() throws Exception {
    IgniteEx node = startGrid(GRID_0);
    node.cluster().state(ClusterState.ACTIVE);
    createEncryptedCache(node, null, cacheName(), null);
    node.encryption().changeCacheGroupKey(Collections.singleton(cacheName())).get();
    IgniteWriteAheadLogManager walMgr = node.context().cache().context().wal();
    long reservedIdx = walMgr.currentSegment();
    assertTrue(walMgr.reserve(new WALPointer(reservedIdx, 0, 0)));
    while (walMgr.lastArchivedSegment() < reservedIdx) {
        long val = ThreadLocalRandom.current().nextLong();
        node.cache(cacheName()).put(val, String.valueOf(val));
    }
    forceCheckpoint();
    int grpId = CU.cacheId(cacheName());
    assertEquals(2, node.context().encryption().groupKeyIds(grpId).size());
    stopAllGrids();
    // Cleanup WAL arcive folder.
    File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
    boolean rmvd = U.delete(new File(dbDir, "wal/archive"));
    assertTrue(rmvd);
    node = startGrid(GRID_0);
    node.cluster().state(ClusterState.ACTIVE);
    while (node.context().encryption().groupKeyIds(grpId).size() != 1) {
        long val = ThreadLocalRandom.current().nextLong();
        node.cache(cacheName()).put(val, String.valueOf(val));
    }
    checkGroupKey(grpId, INITIAL_KEY_ID + 1, MAX_AWAIT_MILLIS);
}

Example 92 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class IgnitePdsReserveWalSegmentsTest method testWalManagerRangeReservation.

/**
 * Tests that range reserved method return correct number of reserved WAL segments.
 *
 * @throws Exception if failed.
 */
@Test
public void testWalManagerRangeReservation() throws Exception {
    IgniteEx n = prepareGrid(2);
    IgniteWriteAheadLogManager wal = n.context().cache().context().wal();
    assertNotNull(wal);
    long resIdx = getReservedWalSegmentIndex(wal);
    assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
    WALPointer lowPtr = lastCheckpointPointer(n);
    assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
    // Reserve previous WAL segment.
    wal.reserve(new WALPointer(resIdx - 1, 0, 0));
    int resCnt = wal.reserved(new WALPointer(resIdx - 1, 0, 0), new WALPointer(resIdx, 0, 0));
    assertTrue("Expected resCnt is 2, real is " + resCnt, resCnt == 2);
}

Example 93 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalScannerTest method shouldDumpToFileAndLogFoundRecord.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldDumpToFileAndLogFoundRecord() throws Exception {
    // given: File for dumping records and test logger for interception of records.
    File targetFile = Paths.get(U.defaultWorkDirectory(), TEST_DUMP_FILE).toFile();
    long expPageId = 984;
    int grpId = 123;
    IgniteLogger log = mock(IgniteLogger.class);
    when(log.isInfoEnabled()).thenReturn(true);
    ArgumentCaptor<String> valCapture = ArgumentCaptor.forClass(String.class);
    doNothing().when(log).info(valCapture.capture());
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024)), new IgniteBiTuple<>(NULL_PTR, new CheckpointRecord(new WALPointer(5738, 0, 0))), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, expPageId, 4)));
    IgniteWalIteratorFactory factory = mock(IgniteWalIteratorFactory.class);
    when(factory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    List<String> actualFileRecords = null;
    try {
        // when: Scanning WAL for searching expected page.
        buildWalScanner(withIteratorParameters(), factory).findAllRecordsFor(groupAndPageIds).forEach(printToLog(log).andThen(printToFile(targetFile)));
        actualFileRecords = Files.readAllLines(targetFile.toPath());
    } finally {
        targetFile.delete();
    }
    actualFileRecords = actualFileRecords.stream().filter(it -> it.startsWith("Next WAL record ::")).collect(Collectors.toList());
    // then: Should be find only expected value from file.
    assertEquals(actualFileRecords.size(), 3);
    assertTrue(actualFileRecords.get(0), actualFileRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualFileRecords.get(1), actualFileRecords.get(1).contains("CheckpointRecord ["));
    assertTrue(actualFileRecords.get(2), actualFileRecords.get(2).contains("FixCountRecord ["));
    // then: Should be find only expected value from log.
    List<String> actualLogRecords = valCapture.getAllValues();
    assertEquals(actualLogRecords.size(), 1);
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("PageSnapshot ["));
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("CheckpointRecord ["));
    assertTrue(actualLogRecords.get(0), actualLogRecords.get(0).contains("FixCountRecord ["));
}

Example 94 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WalScannerTest method shouldFindCorrectRecords.

/**
 * @throws Exception If failed.
 */
@Test
public void shouldFindCorrectRecords() throws Exception {
    // given: Iterator with random value and value which should be find by scanner.
    long expPageId = 984;
    int grpId = 123;
    PageSnapshot expPageSnapshot = new PageSnapshot(new FullPageId(expPageId, grpId), dummyPage(1024, expPageId), 1024);
    CheckpointRecord expCheckpoint = new CheckpointRecord(new WALPointer(5738, 0, 0));
    FixCountRecord expDeltaPage = new FixCountRecord(grpId, expPageId, 4);
    WALIterator mockedIter = mockWalIterator(new IgniteBiTuple<>(NULL_PTR, expPageSnapshot), new IgniteBiTuple<>(NULL_PTR, new PageSnapshot(new FullPageId(455, grpId), dummyPage(4096, 455), 1024)), new IgniteBiTuple<>(NULL_PTR, expCheckpoint), new IgniteBiTuple<>(NULL_PTR, new MetastoreDataRecord("key", new byte[0])), new IgniteBiTuple<>(NULL_PTR, new PartitionMetaStateRecord(grpId, 1, OWNING, 1)), new IgniteBiTuple<>(NULL_PTR, expDeltaPage), new IgniteBiTuple<>(NULL_PTR, new FixCountRecord(grpId, 98348, 4)));
    IgniteWalIteratorFactory mockedFactory = mock(IgniteWalIteratorFactory.class);
    when(mockedFactory.iterator(any(IteratorParametersBuilder.class))).thenReturn(mockedIter);
    // Test scanner handler for holding found value instead of printing its.
    List<WALRecord> holder = new ArrayList<>();
    ScannerHandler recordCaptor = (rec) -> holder.add(rec.get2());
    Set<T2<Integer, Long>> groupAndPageIds = new HashSet<>();
    groupAndPageIds.add(new T2<>(grpId, expPageId));
    // when: Scanning WAL for searching expected page.
    buildWalScanner(withIteratorParameters(), mockedFactory).findAllRecordsFor(groupAndPageIds).forEach(recordCaptor);
    // then: Should be find only expected value.
    assertEquals(holder.size(), 3);
    assertEquals(expPageSnapshot, holder.get(0));
    assertEquals(expCheckpoint, holder.get(1));
    assertEquals(expDeltaPage, holder.get(2));
}

Example 95 with WALPointer

use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.

the class WALRecordSerializationTest method testAllWalRecordsSerializedCompressedAndThenDeserializedSuccessfully.

/**
 * @throws Exception If fail.
 */
@Test
public void testAllWalRecordsSerializedCompressedAndThenDeserializedSuccessfully() throws Exception {
    compactionEnabled = true;
    IgniteEx ignite = startGrid(0);
    ignite.cluster().active(true);
    WALRecord.RecordType[] recordTypes = WALRecord.RecordType.values();
    List<ReflectionEquals> serializedRecords = new ArrayList<>();
    IgniteWriteAheadLogManager wal = ignite.context().cache().context().wal();
    WALPointer lastPointer = null;
    ignite.context().cache().context().database().checkpointReadLock();
    try {
        for (WALRecord.RecordType recordType : recordTypes) {
            WALRecord record = RecordUtils.buildWalRecord(recordType);
            if (RecordUtils.isIncludeIntoLog(record) && (recordType.purpose() == WALRecord.RecordPurpose.LOGICAL || recordType == WALRecord.RecordType.CHECKPOINT_RECORD)) {
                serializedRecords.add(new ReflectionEquals(record, "prev", "pos", // updateCounter for PartitionMetaStateRecord isn't serialized.
                "updateCounter"));
                lastPointer = wal.log(record);
            }
        }
        wal.flush(null, true);
    } finally {
        ignite.context().cache().context().database().checkpointReadUnlock();
    }
    String nodeFolderName = ignite.context().pdsFolderResolver().resolveFolders().folderName();
    File nodeArchiveDir = Paths.get(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false).getAbsolutePath(), "wal", "archive", nodeFolderName).toFile();
    File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(lastPointer.index()));
    File walZipSegment = new File(nodeArchiveDir, FileDescriptor.fileName(lastPointer.index()) + ZIP_SUFFIX);
    // Spam WAL to move all data records to compressible WAL zone.
    for (int i = 0; i < WAL_SEGMENT_SIZE / DFLT_PAGE_SIZE * 2; i++) wal.log(new PageSnapshot(new FullPageId(-1, -1), new byte[DFLT_PAGE_SIZE], 1));
    ignite.getOrCreateCache("generateDirtyPages");
    // WAL archive segment is allowed to be compressed when it's at least one checkpoint away from current WAL head.
    ignite.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
    ignite.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
    for (int i = 0; i < WAL_SEGMENT_SIZE / DFLT_PAGE_SIZE * 2; i++) wal.log(new PageSnapshot(new FullPageId(-1, -1), new byte[DFLT_PAGE_SIZE], 1));
    // Awaiting of zipping of the desirable segment.
    assertTrue(GridTestUtils.waitForCondition(walZipSegment::exists, 15_000));
    // Awaiting of removing of the desirable segment.
    assertTrue(GridTestUtils.waitForCondition(() -> !walSegment.exists(), 15_000));
    stopGrid(0);
    Iterator<ReflectionEquals> serializedIter = serializedRecords.iterator();
    ReflectionEquals curExpRecord = serializedIter.hasNext() ? serializedIter.next() : null;
    try (WALIterator iter = wal.replay(null)) {
        while (iter.hasNext()) {
            WALRecord record = iter.nextX().get2();
            if (curExpRecord != null && curExpRecord.matches(record))
                curExpRecord = serializedIter.hasNext() ? serializedIter.next() : null;
        }
    }
    assertNull("Expected record '" + curExpRecord + "' not found.", curExpRecord);
}