Examples with FileWriteAheadLogManager org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager used on opensource projects

Example 1 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class IgnitePdsDiskErrorsRecoveringTest method emulateRecoveringOnWALWritingError.

/**
 */
private void emulateRecoveringOnWALWritingError() throws Exception {
    final IgniteEx grid = startGrid(0);
    FileWriteAheadLogManager wal = (FileWriteAheadLogManager) grid.context().cache().context().wal();
    wal.setFileIOFactory(new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), diskSpaceBytes));
    grid.active(true);
    int failedPosition = -1;
    for (int i = 0; i < 1000; i++) {
        byte payload = (byte) i;
        byte[] data = new byte[2048];
        Arrays.fill(data, payload);
        try {
            grid.cache(CACHE_NAME).put(i, data);
        } catch (Exception e) {
            failedPosition = i;
            break;
        }
    }
    // We must be able to put something into cache before fail.
    Assert.assertTrue(failedPosition > 0);
    // Grid should be automatically stopped after WAL fail.
    awaitStop(grid);
    // Grid should be successfully recovered after stopping.
    IgniteEx recoveredGrid = startGrid(0);
    recoveredGrid.active(true);
    for (int i = 0; i < failedPosition; i++) {
        byte payload = (byte) i;
        byte[] data = new byte[2048];
        Arrays.fill(data, payload);
        byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
        Assert.assertArrayEquals(data, actualData);
    }
}

Example 2 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class CdcCacheVersionTest method testConflictVersionWritten.

/**
 * Test that conflict version is writtern to WAL.
 */
@Test
public void testConflictVersionWritten() throws Exception {
    walProvider = (ctx) -> new FileWriteAheadLogManager(ctx) {

        @Override
        public WALPointer log(WALRecord rec) throws IgniteCheckedException {
            if (rec.type() != DATA_RECORD_V2)
                return super.log(rec);
            DataRecord dataRec = (DataRecord) rec;
            for (int i = 0; i < dataRec.entryCount(); i++) {
                DataEntry dataEntry = dataRec.writeEntries().get(i);
                assertEquals(CU.cacheId(DEFAULT_CACHE_NAME), dataEntry.cacheId());
                assertEquals(DFLT_CLUSTER_ID, dataEntry.writeVersion().dataCenterId());
                assertNotNull(dataEntry.writeVersion().conflictVersion());
                assertEquals(OTHER_CLUSTER_ID, dataEntry.writeVersion().conflictVersion().dataCenterId());
                walRecCheckedCntr.incrementAndGet();
            }
            return super.log(rec);
        }
    };
    conflictResolutionMgrSupplier = () -> new CacheVersionConflictResolver() {

        @Override
        public <K1, V1> GridCacheVersionConflictContext<K1, V1> resolve(CacheObjectValueContext ctx, GridCacheVersionedEntryEx<K1, V1> oldEntry, GridCacheVersionedEntryEx<K1, V1> newEntry, boolean atomicVerComparator) {
            GridCacheVersionConflictContext<K1, V1> res = new GridCacheVersionConflictContext<>(ctx, oldEntry, newEntry);
            res.useNew();
            assertEquals(OTHER_CLUSTER_ID, newEntry.version().dataCenterId());
            if (!oldEntry.isStartVersion())
                assertEquals(OTHER_CLUSTER_ID, oldEntry.version().dataCenterId());
            conflictCheckedCntr.incrementAndGet();
            return res;
        }

        @Override
        public String toString() {
            return "TestCacheConflictResolutionManager";
        }
    };
    startGrids(gridCnt);
    IgniteEx cli = startClientGrid(gridCnt);
    for (int i = 0; i < gridCnt; i++) {
        grid(i).context().cache().context().versions().dataCenterId(DFLT_CLUSTER_ID);
        assertEquals(DFLT_CLUSTER_ID, grid(i).context().metric().registry(CACHE_METRICS).<IntMetric>findMetric(DATA_VER_CLUSTER_ID).value());
    }
    cli.cluster().state(ACTIVE);
    IgniteCache<Integer, User> cache = cli.getOrCreateCache(new CacheConfiguration<Integer, User>(DEFAULT_CACHE_NAME).setCacheMode(cacheMode).setAtomicityMode(atomicityMode).setBackups(Integer.MAX_VALUE));
    if (atomicityMode == ATOMIC)
        putRemoveCheck(cli, cache, null, null);
    else {
        // Check operations for transaction cache without explicit transaction.
        putRemoveCheck(cli, cache, null, null);
        // Check operations for transaction cache with explicit transaction in all modes.
        for (TransactionConcurrency concurrency : TransactionConcurrency.values()) for (TransactionIsolation isolation : TransactionIsolation.values()) putRemoveCheck(cli, cache, concurrency, isolation);
    }
    for (int i = 0; i < gridCnt; i++) {
        boolean dfltCacheFound = false;
        assertFalse(grid(i).context().clientNode());
        SystemView<CacheView> caches = grid(i).context().systemView().view(CACHES_VIEW);
        for (CacheView v : caches) {
            if (v.cacheName().equals(DEFAULT_CACHE_NAME)) {
                assertEquals(v.conflictResolver(), "TestCacheConflictResolutionManager");
                dfltCacheFound = true;
            } else
                assertNull(v.conflictResolver());
        }
        assertTrue(dfltCacheFound);
    }
}

Example 3 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class IgnitePdsReserveWalSegmentsTest method testMinReserveIdx.

/**
 * Check that the minimum reserved index will not be greater than the actual deleted segment.
 *
 * @throws Exception If failed.
 */
@Test
public void testMinReserveIdx() throws Exception {
    IgniteEx n = prepareGrid(1);
    forceCheckpoint();
    FileWriteAheadLogManager wal = (FileWriteAheadLogManager) n.context().cache().context().wal();
    assertNotNull(wal);
    if (compactionEnabled(n))
        assertTrue(waitForCondition(() -> wal.lastCompactedSegment() >= 1, 10_000));
    assertEquals(1, wal.truncate(new WALPointer(1, 0, 0)));
    Long minReserveIdx = getFieldValueHierarchy(wal, "segmentAware", "reservationStorage", "minReserveIdx");
    assertEquals(0L, minReserveIdx.longValue());
}

Example 4 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class IgnitePdsStartWIthEmptyArchive method test.

/**
 * @throws Exception If failed.
 */
@Test
public void test() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.cluster().active(true);
    FileWriteAheadLogManager walMgr = (FileWriteAheadLogManager) ig.context().cache().context().wal();
    // Populate data for generate WAL archive segments.
    try (IgniteDataStreamer<Integer, byte[]> st = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
        int entries = 1000;
        for (int i = 0; i < entries; i++) st.addData(i, new byte[1024 * 1024]);
    }
    File archiveDir = U.field(walMgr, "walArchiveDir");
    stopGrid(0, false);
    SegmentAware beforeSaw = U.field(walMgr, "segmentAware");
    long beforeLastArchivedAbsoluteIdx = beforeSaw.lastArchivedAbsoluteIndex();
    FileWriteHandle fhBefore = U.field(walMgr, "currHnd");
    long idxBefore = fhBefore.getSegmentId();
    File[] files = archiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER);
    Arrays.sort(files);
    // Cleanup archive directory.
    for (File f : files) {
        if (f.delete())
            log.info("File " + f.getAbsolutePath() + " deleted");
    }
    Assert.assertEquals(0, archiveDir.listFiles().length);
    evts.clear();
    // Restart grid again after archive was removed.
    ig = startGrid(0);
    walMgr = (FileWriteAheadLogManager) ig.context().cache().context().wal();
    SegmentAware afterSaw = U.field(walMgr, "segmentAware");
    long afterLastArchivedAbsoluteIndex = afterSaw.lastArchivedAbsoluteIndex();
    int segments = ig.configuration().getDataStorageConfiguration().getWalSegments();
    Assert.assertTrue("lastArchivedBeforeIdx=" + beforeLastArchivedAbsoluteIdx + ", lastArchivedAfterIdx=" + afterLastArchivedAbsoluteIndex + ",  segments=" + segments, afterLastArchivedAbsoluteIndex >= (beforeLastArchivedAbsoluteIdx - segments));
    ig.cluster().active(true);
    FileWriteHandle fhAfter = U.field(walMgr, "currHnd");
    Assert.assertNotNull(fhAfter);
    long idxAfter = fhAfter.getSegmentId();
    Assert.assertEquals(idxBefore, idxAfter);
    Assert.assertTrue(idxAfter >= beforeLastArchivedAbsoluteIdx);
    log.info("currentIdx=" + idxAfter + ", lastArchivedBeforeIdx=" + beforeLastArchivedAbsoluteIdx + ", lastArchivedAfteridx=" + afterLastArchivedAbsoluteIndex + ",  segments=" + segments);
    // One is a last archived, secod is a current write segment.
    final long awaitAchviedSegments = idxAfter - afterLastArchivedAbsoluteIndex - 2;
    // Await all current available semgment will be archived.
    assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            long cut = evts.keySet().stream().filter(e -> e > afterLastArchivedAbsoluteIndex).count();
            return cut >= awaitAchviedSegments;
        }
    }, 10_000));
}

Example 5 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class LinuxNativeIoPluginProvider method setupDirect.

/**
 * @param ignite Ignite starting up.
 * @return Managed aligned buffers and its associated threads. This collection is used to free buffers. May return
 * {@code null}.
 */
@Nullable
private ConcurrentHashMap<Long, Thread> setupDirect(IgniteEx ignite) {
    GridCacheSharedContext<Object, Object> cacheCtx = ignite.context().cache().context();
    IgnitePageStoreManager ignitePageStoreMgr = cacheCtx.pageStore();
    if (ignitePageStoreMgr == null)
        return null;
    if (!(ignitePageStoreMgr instanceof FilePageStoreManager))
        return null;
    final FilePageStoreManager pageStore = (FilePageStoreManager) ignitePageStoreMgr;
    FileIOFactory backupIoFactory = pageStore.getPageStoreFileIoFactory();
    final AlignedBuffersDirectFileIOFactory factory = new AlignedBuffersDirectFileIOFactory(ignite.log(), pageStore.workDir(), pageStore.pageSize(), backupIoFactory);
    final IgniteWriteAheadLogManager walMgr = cacheCtx.wal();
    if (walMgr != null && walMgr instanceof FileWriteAheadLogManager && IgniteNativeIoLib.isJnaAvailable()) {
        ((FileWriteAheadLogManager) walMgr).setCreateWalFileListener(new IgniteInClosure<FileIO>() {

            @Override
            public void apply(FileIO fileIO) {
                adviceFileDontNeed(fileIO, ((FileWriteAheadLogManager) walMgr).maxWalSegmentSize());
            }
        });
    }
    if (!factory.isDirectIoAvailable())
        return null;
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) cacheCtx.database();
    db.setThreadBuf(new ThreadLocal<ByteBuffer>() {

        @Override
        protected ByteBuffer initialValue() {
            return factory.createManagedBuffer(pageStore.pageSize());
        }
    });
    pageStore.setPageStoreFileIOFactories(factory, backupIoFactory);
    return factory.managedAlignedBuffers();
}