Examples with FileDescriptor org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor used on opensource projects

Example 21 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWalRebalanceTest method testSwitchHistoricalRebalanceToFullWhileIteratingOverWAL.

/**
 * Tests that demander switches to full rebalance if the previously chosen supplier for a group has failed
 * to perform historical rebalance due to an unexpected error while iterating over reserved wal.
 *
 * @throws Exception If failed
 */
@Test
public void testSwitchHistoricalRebalanceToFullWhileIteratingOverWAL() throws Exception {
    testSwitchHistoricalRebalanceToFull(supplier1 -> {
        try {
            // Corrupt wal record in order to fail historical rebalance from supplier1 node.
            IgniteWriteAheadLogManager walMgr = supplier1.context().cache().context().wal();
            WALPointer ptr = walMgr.log(new DataRecord(new DataEntry(CU.cacheId("test-cache-1"), new KeyCacheObjectImpl(0, null, 0), null, GridCacheOperation.DELETE, new GridCacheVersion(0, 1, 1, 0), new GridCacheVersion(0, 1, 1, 0), 0, 0, 0, DataEntry.EMPTY_FLAGS)));
            File walDir = U.field(walMgr, "walWorkDir");
            List<FileDescriptor> walFiles = new IgniteWalIteratorFactory().resolveWalFiles(new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(walDir));
            FileDescriptor lastWalFile = walFiles.get(walFiles.size() - 1);
            WalTestUtils.corruptWalSegmentFile(lastWalFile, ptr);
            IgniteCache<Integer, IndexedObject> c1 = supplier1.cache("test-cache-1");
            for (int i = 0; i < PARTS_CNT * 100; i++) c1.put(i, new IndexedObject(i + PARTS_CNT));
        } catch (IgniteCheckedException | IOException e) {
            throw new RuntimeException(e);
        }
    }, () -> true);
}

Example 22 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteWithoutArchiverWalIteratorInvalidCrcTest method nodeShouldStartIfBinaryRecordCorruptedBeforeEndCheckpoint.

/**
 *  Binary record was corrupted, before start last checkpoint without end.
 *  -----||--X---||--------->
 *  Node can't start.
 */
@Test
public void nodeShouldStartIfBinaryRecordCorruptedBeforeEndCheckpoint() throws Exception {
    startNodeAndPopulate();
    stopGrid(0, true);
    IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
    File walDir = U.field(walMgr, "walWorkDir");
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    List<FileDescriptor> walFiles = getWalFiles(walDir, iterFactory);
    FileDescriptor lastWalFile = walFiles.get(walFiles.size() - 1);
    List<WALPointer> checkpoints = WalTestUtils.getPointers(lastWalFile, iterFactory, CHECKPOINT_RECORD);
    List<WALPointer> binary = WalTestUtils.getPointers(lastWalFile, iterFactory, PHYSICAL).stream().filter(p -> p.fileOffset() < checkpoints.get(checkpoints.size() - 1).fileOffset()).collect(Collectors.toList());
    WALPointer pointer = binary.get(binary.size() - 1);
    WalTestUtils.corruptWalSegmentFile(lastWalFile, pointer);
    GridTestUtils.assertThrows(log, () -> startGrid(0), Exception.class, null);
}

Example 23 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class WriteAheadLogManagerSelfTest method testReservation.

/**
 * Checking the correctness of WAL segment reservation.
 *
 * @throws Exception If failed.
 */
@Test
public void testReservation() throws Exception {
    IgniteEx n = startGrids(1);
    for (int i = 0; walMgr(n).lastArchivedSegment() < 2; i++) n.cache(DEFAULT_CACHE_NAME).put(i, new byte[(int) (10 * U.KB)]);
    forceCheckpoint();
    assertTrue(walMgr(n).lastArchivedSegment() >= 2);
    assertTrue(walMgr(n).lastTruncatedSegment() == -1);
    WALPointer segment0WalPtr = new WALPointer(0, 0, 0);
    assertTrue(walMgr(n).reserve(segment0WalPtr));
    assertTrue(walMgr(n).reserved(segment0WalPtr));
    WALPointer segment1WalPtr = new WALPointer(1, 0, 0);
    // Delete segment manually.
    FileDescriptor segment1 = Arrays.stream(walMgr(n).walArchiveFiles()).filter(fd -> fd.idx() == segment1WalPtr.index()).findAny().orElseThrow(AssertionError::new);
    assertTrue(segment1.file().delete());
    assertFalse(walMgr(n).reserve(segment1WalPtr));
    assertTrue(walMgr(n).reserved(segment1WalPtr));
    walMgr(n).release(segment0WalPtr);
    assertFalse(walMgr(n).reserved(segment0WalPtr));
    assertFalse(walMgr(n).reserved(segment1WalPtr));
    assertEquals(1, walMgr(n).truncate(segment1WalPtr));
    assertFalse(walMgr(n).reserve(segment0WalPtr));
    assertFalse(walMgr(n).reserve(segment1WalPtr));
    assertFalse(walMgr(n).reserved(segment0WalPtr));
    assertFalse(walMgr(n).reserved(segment1WalPtr));
    WALPointer segmentMaxWalPtr = new WALPointer(Long.MAX_VALUE, 0, 0);
    assertFalse(walMgr(n).reserve(segmentMaxWalPtr));
    assertFalse(walMgr(n).reserved(segmentMaxWalPtr));
}

Example 24 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class IgniteAbstractWalIteratorInvalidCrcTest method doTest.

/**
 * @param descPicker Function that picks WAL segment to corrupt from archive segments list
 *      and working directory segments list.
 * @param ignoreArchiveDir Do not iterate over archive segments if this flag is true.
 * @param shouldFail Whether iteration is axpected to fail or not.
 * @throws IOException If IO exception.
 * @throws IgniteCheckedException If iterator failed.
 */
protected void doTest(BiFunction<List<FileDescriptor>, List<FileDescriptor>, FileDescriptor> descPicker, boolean ignoreArchiveDir, boolean shouldFail) throws IOException, IgniteCheckedException {
    IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
    IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
    File walArchiveDir = U.field(walMgr, "walArchiveDir");
    List<FileDescriptor> archiveDescs = iterFactory.resolveWalFiles(new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(walArchiveDir));
    File walDir = U.field(walMgr, "walWorkDir");
    List<FileDescriptor> descs = iterFactory.resolveWalFiles(new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(walDir));
    FileDescriptor corruptedDesc = descPicker.apply(archiveDescs, descs);
    WALPointer beforeCorruptedPtr = WalTestUtils.corruptWalSegmentFile(corruptedDesc, iterFactory, random);
    if (shouldFail) {
        WALPointer[] lastReadPtrRef = new WALPointer[1];
        IgniteException igniteException = (IgniteException) GridTestUtils.assertThrows(log, () -> {
            try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
                for (IgniteBiTuple<WALPointer, WALRecord> tuple : iter) {
                    WALPointer ptr = tuple.get1();
                    lastReadPtrRef[0] = ptr;
                }
            }
            return null;
        }, IgniteException.class, "Failed to read WAL record");
        assertTrue(igniteException.hasCause(IgniteDataIntegrityViolationException.class));
        WALPointer lastReadPtr = lastReadPtrRef[0];
        assertNotNull(lastReadPtr);
        // WAL iterator advances to the next record and only then returns current one,
        // so next record has to be valid as well.
        assertEquals(lastReadPtr, beforeCorruptedPtr);
    } else
        try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
            while (iter.hasNext()) iter.next();
        }
}

Example 25 with FileDescriptor

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor in project ignite by apache.

the class WalDeletionArchiveAbstractTest method testCorrectDeletedCheckpointHistoryButKeepWalFiles.

/**
 * Correct delete checkpoint history from memory depends on IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE.
 * WAL files doesn't delete because deleting was disabled.
 */
@Test
@WithSystemProperty(key = IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, value = "2")
public void testCorrectDeletedCheckpointHistoryButKeepWalFiles() throws Exception {
    // given: configured grid with disabled WAL removing.
    Ignite ignite = startGrid(dbCfg -> dbCfg.setMaxWalArchiveSize(DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE));
    GridCacheDatabaseSharedManager dbMgr = gridDatabase(ignite);
    CheckpointHistory hist = dbMgr.checkpointHistory();
    assertNotNull(hist);
    IgniteCache<Integer, Object> cache = ignite.getOrCreateCache(cacheConfiguration());
    // when: put to cache
    for (int i = 0; i < 500; i++) {
        cache.put(i, i);
        if (i % 10 == 0)
            forceCheckpoint();
    }
    forceCheckpoint();
    // then: WAL files was not deleted but some of checkpoint history was deleted.
    FileWriteAheadLogManager wal = wal(ignite);
    assertNull(getFieldValueHierarchy(wal, "cleaner"));
    FileDescriptor[] files = wal.walArchiveFiles();
    assertTrue(Stream.of(files).anyMatch(desc -> desc.file().getName().endsWith("0001.wal")));
    assertTrue(hist.checkpoints().size() == 2);
}