Examples with FileDescriptor org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor used on opensource projects

Example 31 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project gridgain by gridgain.

the class CorruptedCheckpointReservationTest method getFileDescriptor.

/**
 * @param segmentCompressed Segment compressed.
 * @param walMgr Wal manager.
 * @param corruptedCp Corrupted checkpoint.
 */
@NotNull
private Optional<FileDescriptor> getFileDescriptor(boolean segmentCompressed, IgniteWriteAheadLogManager walMgr, FileWALPointer corruptedCp) {
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    File walArchiveDir = U.field(walMgr, "walArchiveDir");
    List<FileDescriptor> walFiles = getWalFiles(walArchiveDir, iterFactory);
    String suffix = segmentCompressed ? FilePageStoreManager.ZIP_SUFFIX : FileDescriptor.WAL_SEGMENT_FILE_EXT;
    return walFiles.stream().filter(w -> w.idx() == corruptedCp.index() && w.file().getName().endsWith(suffix)).findFirst();
}

Example 32 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project gridgain by gridgain.

the class WriteAheadLogManagerSelfTest method testReservation.

/**
 * Checking the correctness of WAL segment reservation.
 *
 * @throws Exception If failed.
 */
@Test
public void testReservation() throws Exception {
    IgniteEx n = startGrids(1);
    for (int i = 0; walMgr(n).lastArchivedSegment() < 2; i++) n.cache(DEFAULT_CACHE_NAME).put(i, new byte[(int) (10 * U.KB)]);
    forceCheckpoint();
    assertTrue(walMgr(n).lastArchivedSegment() >= 2);
    assertTrue(walMgr(n).lastTruncatedSegment() == -1);
    FileWALPointer segment0WalPtr = new FileWALPointer(0, 0, 0);
    assertTrue(walMgr(n).reserve(segment0WalPtr));
    assertTrue(walMgr(n).reserved(segment0WalPtr));
    FileWALPointer segment1WalPtr = new FileWALPointer(1, 0, 0);
    // Delete segment manually.
    FileDescriptor segment1 = Arrays.stream(walMgr(n).walArchiveFiles()).filter(fd -> fd.idx() == segment1WalPtr.index()).findAny().orElseThrow(AssertionError::new);
    assertTrue(segment1.file().delete());
    assertFalse(walMgr(n).reserve(segment1WalPtr));
    assertTrue(walMgr(n).reserved(segment1WalPtr));
    walMgr(n).release(segment0WalPtr);
    assertFalse(walMgr(n).reserved(segment0WalPtr));
    assertFalse(walMgr(n).reserved(segment1WalPtr));
    assertEquals(1, walMgr(n).truncate(segment1WalPtr));
    assertFalse(walMgr(n).reserve(segment0WalPtr));
    assertFalse(walMgr(n).reserve(segment1WalPtr));
    assertFalse(walMgr(n).reserved(segment0WalPtr));
    assertFalse(walMgr(n).reserved(segment1WalPtr));
    FileWALPointer segmentMaxWalPtr = new FileWALPointer(Long.MAX_VALUE, 0, 0);
    assertFalse(walMgr(n).reserve(segmentMaxWalPtr));
    assertFalse(walMgr(n).reserved(segmentMaxWalPtr));
}

Example 33 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project gridgain by gridgain.

the class IgniteAbstractWalIteratorInvalidCrcTest method doTest.

/**
 * @param descPicker Function that picks WAL segment to corrupt from archive segments list
 *      and working directory segments list.
 * @param ignoreArchiveDir Do not iterate over archive segments if this flag is true.
 * @param shouldFail Whether iteration is axpected to fail or not.
 * @throws IOException If IO exception.
 * @throws IgniteCheckedException If iterator failed.
 */
protected void doTest(BiFunction<List<FileDescriptor>, List<FileDescriptor>, FileDescriptor> descPicker, boolean ignoreArchiveDir, boolean shouldFail) throws IOException, IgniteCheckedException {
    IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    File walArchiveDir = U.field(walMgr, "walArchiveDir");
    List<FileDescriptor> archiveDescs = iterFactory.resolveWalFiles(new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(walArchiveDir));
    File walDir = U.field(walMgr, "walWorkDir");
    List<FileDescriptor> descs = iterFactory.resolveWalFiles(new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(walDir));
    FileDescriptor corruptedDesc = descPicker.apply(archiveDescs, descs);
    FileWALPointer beforeCorruptedPtr = WalTestUtils.corruptRandomWalRecord(corruptedDesc, iterFactory, random);
    if (shouldFail) {
        FileWALPointer[] lastReadPtrRef = new FileWALPointer[1];
        IgniteException igniteException = (IgniteException) GridTestUtils.assertThrows(log, () -> {
            try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
                for (IgniteBiTuple<WALPointer, WALRecord> tuple : iter) {
                    FileWALPointer ptr = (FileWALPointer) tuple.get1();
                    lastReadPtrRef[0] = ptr;
                }
            }
            return null;
        }, IgniteException.class, "Failed to read WAL record");
        assertTrue(igniteException.hasCause(IgniteDataIntegrityViolationException.class));
        FileWALPointer lastReadPtr = lastReadPtrRef[0];
        assertNotNull(lastReadPtr);
        // WAL iterator advances to the next record and only then returns current one,
        // so next record has to be valid as well.
        assertEquals(lastReadPtr, beforeCorruptedPtr);
    } else
        try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
            while (iter.hasNext()) iter.next();
        }
}

Example 34 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project gridgain by gridgain.

the class WalDeletionArchiveAbstractTest method testCorrectDeletedArchivedWalFiles.

/**
 * Correct delete archived wal files.
 */
@Test
public void testCorrectDeletedArchivedWalFiles() throws Exception {
    // given: configured grid with setted max wal archive size
    long maxWalArchiveSize = 2 * 1024 * 1024;
    Ignite ignite = startGrid(dbCfg -> dbCfg.setMaxWalArchiveSize(maxWalArchiveSize));
    GridCacheDatabaseSharedManager dbMgr = gridDatabase(ignite);
    CheckpointHistory hist = dbMgr.checkpointHistory();
    assertNotNull(hist);
    IgniteCache<Integer, Object> cache = ignite.getOrCreateCache(cacheConfiguration());
    // when: put to cache more than 2 MB
    for (int i = 0; i < 500; i++) {
        if (i % 100 == 0)
            forceCheckpoint();
        cache.put(i, i);
    }
    // then: total archive size less than of maxWalArchiveSize(by current logic)
    FileWriteAheadLogManager wal = wal(ignite);
    assertTrue(waitForCondition(() -> wal.lastTruncatedSegment() >= 0, 10_000));
    FileDescriptor[] files = wal.walArchiveFiles();
    long totalSize = wal.totalSize(files);
    assertTrue(files.length >= 1);
    assertTrue(totalSize < maxWalArchiveSize);
    assertFalse(Stream.of(files).anyMatch(desc -> desc.file().getName().endsWith("00001.wal")));
    assertTrue(!hist.checkpoints().isEmpty());
}

Example 35 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project gridgain by gridgain.

the class WalDeletionArchiveAbstractTest method testCorrectDeletedCheckpointHistoryButKeepWalFiles.

/**
 * Correct delete checkpoint history from memory depends on IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE.
 * WAL files doesn't delete because deleting was disabled.
 */
@Test
@WithSystemProperty(key = IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, value = "2")
public void testCorrectDeletedCheckpointHistoryButKeepWalFiles() throws Exception {
    // given: configured grid with disabled WAL removing.
    Ignite ignite = startGrid(dbCfg -> dbCfg.setMaxWalArchiveSize(DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE));
    GridCacheDatabaseSharedManager dbMgr = gridDatabase(ignite);
    CheckpointHistory hist = dbMgr.checkpointHistory();
    assertNotNull(hist);
    IgniteCache<Integer, Object> cache = ignite.getOrCreateCache(cacheConfiguration());
    // when: put to cache
    for (int i = 0; i < 500; i++) {
        cache.put(i, i);
        if (i % 10 == 0)
            forceCheckpoint();
    }
    forceCheckpoint();
    // then: WAL files was not deleted but some of checkpoint history was deleted.
    FileWriteAheadLogManager wal = wal(ignite);
    assertNull(getFieldValueHierarchy(wal, "cleaner"));
    FileDescriptor[] files = wal.walArchiveFiles();
    assertTrue(Stream.of(files).anyMatch(desc -> desc.file().getName().endsWith("0001.wal")));
    assertTrue(hist.checkpoints().size() == 2);
}