Examples with FileWriteAheadLogManager org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager used on opensource projects

Example 6 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class MockWalIteratorFactory method iterator.

/**
 * Creates iterator
 * @param wal WAL directory without node consistent id
 * @param walArchive WAL archive without node consistent id
 * @return iterator
 * @throws IgniteCheckedException if IO failed
 */
@SuppressWarnings("unchecked")
public WALIterator iterator(File wal, File walArchive) throws IgniteCheckedException {
    final DataStorageConfiguration persistentCfg1 = Mockito.mock(DataStorageConfiguration.class);
    when(persistentCfg1.getWalPath()).thenReturn(wal.getAbsolutePath());
    when(persistentCfg1.getWalArchivePath()).thenReturn(walArchive.getAbsolutePath());
    when(persistentCfg1.getWalSegments()).thenReturn(segments);
    when(persistentCfg1.getWalBufferSize()).thenReturn(DataStorageConfiguration.DFLT_WAL_BUFF_SIZE);
    when(persistentCfg1.getWalRecordIteratorBufferSize()).thenReturn(DataStorageConfiguration.DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE);
    final FileIOFactory fileIOFactory = new DataStorageConfiguration().getFileIOFactory();
    when(persistentCfg1.getFileIOFactory()).thenReturn(fileIOFactory);
    final IgniteConfiguration cfg = Mockito.mock(IgniteConfiguration.class);
    when(cfg.getDataStorageConfiguration()).thenReturn(persistentCfg1);
    final GridKernalContext ctx = Mockito.mock(GridKernalContext.class);
    when(ctx.config()).thenReturn(cfg);
    when(ctx.clientNode()).thenReturn(false);
    when(ctx.pdsFolderResolver()).thenReturn(new PdsFoldersResolver() {

        @Override
        public PdsFolderSettings resolveFolders() {
            return new PdsFolderSettings(new File("."), subfolderName, consistentId, null, false);
        }
    });
    final GridDiscoveryManager disco = Mockito.mock(GridDiscoveryManager.class);
    when(ctx.discovery()).thenReturn(disco);
    final IgniteWriteAheadLogManager mgr = new FileWriteAheadLogManager(ctx);
    final GridCacheSharedContext sctx = Mockito.mock(GridCacheSharedContext.class);
    when(sctx.kernalContext()).thenReturn(ctx);
    when(sctx.discovery()).thenReturn(disco);
    when(sctx.gridConfig()).thenReturn(cfg);
    final GridCacheDatabaseSharedManager db = Mockito.mock(GridCacheDatabaseSharedManager.class);
    when(db.pageSize()).thenReturn(pageSize);
    when(sctx.database()).thenReturn(db);
    when(sctx.logger(any(Class.class))).thenReturn(log);
    mgr.start(sctx);
    return mgr.replay(null);
}

Example 7 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class ProgressWatchdog method tick.

/**
 * Regular method printing statistics to out and to log. Checks gaps in progress.
 */
private void tick() {
    long elapsedMs = U.currentTimeMillis() - msStart;
    final long totalCnt = overallRecordsProcessed.longValue();
    long elapsedMsFromPrevTick = elapsedMs - prevMsElapsed.getAndSet(elapsedMs);
    if (elapsedMsFromPrevTick == 0)
        return;
    final long currPutPerSec = ((totalCnt - prevRecordsCnt.getAndSet(totalCnt)) * 1000) / elapsedMsFromPrevTick;
    final long averagePutPerSec = totalCnt * 1000 / elapsedMs;
    boolean slowProgress = currPutPerSec < averagePutPerSec / 10 && !stopping;
    final String fileNameWithDump = slowProgress ? reactNoProgress() : "";
    DataStorageConfiguration dsCfg = ignite.configuration().getDataStorageConfiguration();
    String defRegName = dsCfg.getDefaultDataRegionConfiguration().getName();
    long dirtyPages = -1;
    for (DataRegionMetrics m : ignite.dataRegionMetrics()) if (m.getName().equals(defRegName))
        dirtyPages = m.getDirtyPages();
    GridCacheSharedContext<Object, Object> cacheSctx = null;
    PageMemoryImpl pageMemory = null;
    try {
        cacheSctx = ((IgniteEx) ignite).context().cache().context();
        pageMemory = (PageMemoryImpl) cacheSctx.database().dataRegion(defRegName).pageMemory();
    } catch (IgniteCheckedException e) {
        e.printStackTrace();
    }
    long cpBufPages = 0;
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (cacheSctx.database());
    AtomicInteger wrPageCntr = db.writtenPagesCounter();
    long cpWrittenPages = wrPageCntr == null ? 0 : wrPageCntr.get();
    AtomicInteger syncedPagesCntr = db.syncedPagesCounter();
    int cpSyncedPages = syncedPagesCntr == null ? 0 : syncedPagesCntr.get();
    AtomicInteger evictedPagesCntr = db.evictedPagesCntr();
    int cpEvictedPages = evictedPagesCntr == null ? 0 : evictedPagesCntr.get();
    int pageSize = pageMemory == null ? 0 : pageMemory.pageSize();
    String cpWriteSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpWrittenPages, prevCpWrittenPages) * pageSize);
    String cpSyncSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpSyncedPages, prevCpSyncedPages) * pageSize);
    String walSpeed = "";
    long throttleParkTimeNanos = 0;
    double curDirtyRatio = 0.0;
    String targetDirtyRatioStr = "";
    double closeToThrottle = 0.0;
    long idx = -1;
    long lastArchIdx = -1;
    int walArchiveSegments = 0;
    long walWorkSegments = 0;
    long markDirtySpeed = 0;
    long cpWriteSpeedInPages = 0;
    long estWrAllSpeed = 0;
    try {
        if (pageMemory != null) {
            cpBufPages = pageMemory.checkpointBufferPagesCount();
            PagesWriteSpeedBasedThrottle throttle = U.field(pageMemory, "writeThrottle");
            if (throttle != null) {
                curDirtyRatio = throttle.getCurrDirtyRatio();
                double targetDirtyRatio = throttle.getTargetDirtyRatio();
                targetDirtyRatioStr = targetDirtyRatio < 0 ? "" : formatDbl(targetDirtyRatio);
                closeToThrottle = throttle.throttleWeight();
                throttleParkTimeNanos = throttle.throttleParkTime();
                markDirtySpeed = throttle.getMarkDirtySpeed();
                cpWriteSpeedInPages = throttle.getCpWriteSpeed();
                estWrAllSpeed = throttle.getLastEstimatedSpeedForMarkAll();
                if (estWrAllSpeed > 99_999)
                    estWrAllSpeed = 99_999;
            }
        }
        FileWriteAheadLogManager wal = (FileWriteAheadLogManager) cacheSctx.wal();
        idx = 0;
        lastArchIdx = 0;
        walArchiveSegments = wal.walArchiveSegments();
        walWorkSegments = idx - lastArchIdx;
        /* // uncomment when currentWritePointer is available
             FileWALPointer ptr = wal.currentWritePointer();
               FileWALPointer prevWalPtr = this.prevWalPtrRef.getAndSet(ptr);

               if (prevWalPtr != null) {
                   long idxDiff = ptr.index() - prevWalPtr.index();
                   long offDiff = ptr.fileOffset() - prevWalPtr.fileOffset();
                   long bytesDiff = idxDiff * maxWalSegmentSize + offDiff;

                   long bytesPerSec = (bytesDiff * 1000) / elapsedMsFromPrevTick;

                   walSpeed = getMBytesPrintable(bytesPerSec);
               } else
                   walSpeed = "0";
             */
        walSpeed = "0";
    } catch (Exception e) {
        X.error(e.getClass().getSimpleName() + ":" + e.getMessage());
    }
    long elapsedSecs = elapsedMs / 1000;
    X.println(" >> " + operation + " done: " + totalCnt + "/" + elapsedSecs + "s, " + "Cur. " + operation + " " + currPutPerSec + " recs/sec " + "cpWriteSpeed=" + cpWriteSpeed + " " + "cpSyncSpeed=" + cpSyncSpeed + " " + "walSpeed= " + walSpeed + " " + "walWorkSeg.=" + walWorkSegments + " " + "markDirtySpeed=" + markDirtySpeed + " " + "Avg. " + operation + " " + averagePutPerSec + " recs/sec, " + "dirtyP=" + dirtyPages + ", " + "cpWrittenP.=" + cpWrittenPages + ", " + "cpBufP.=" + cpBufPages + " " + "threshold=" + targetDirtyRatioStr + " " + "walIdx=" + idx + " " + "archWalIdx=" + lastArchIdx + " " + "walArchiveSegments=" + walArchiveSegments + " " + fileNameWithDump);
    line(elapsedSecs, currPutPerSec, walSpeed, cpWriteSpeed, cpSyncSpeed, walWorkSegments, throttleParkTimeNanos, formatDbl(curDirtyRatio), targetDirtyRatioStr, formatDbl(closeToThrottle), markDirtySpeed, cpWriteSpeedInPages, estWrAllSpeed, averagePutPerSec, dirtyPages, cpWrittenPages, cpSyncedPages, cpEvictedPages, idx, lastArchIdx, walArchiveSegments);
}

Example 8 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class GridCacheProcessor method createSharedContext.

/**
 * Creates shared context.
 *
 * @param kernalCtx Kernal context.
 * @param storeSesLsnrs Store session listeners.
 * @return Shared context.
 * @throws IgniteCheckedException If failed.
 */
@SuppressWarnings("unchecked")
private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx, Collection<CacheStoreSessionListener> storeSesLsnrs) throws IgniteCheckedException {
    IgniteTxManager tm = new IgniteTxManager();
    GridCacheMvccManager mvccMgr = new GridCacheMvccManager();
    GridCacheVersionManager verMgr = new GridCacheVersionManager();
    GridCacheDeploymentManager depMgr = new GridCacheDeploymentManager();
    GridCachePartitionExchangeManager exchMgr = new GridCachePartitionExchangeManager();
    IgniteCacheDatabaseSharedManager dbMgr;
    IgnitePageStoreManager pageStoreMgr = null;
    IgniteWriteAheadLogManager walMgr = null;
    if (CU.isPersistenceEnabled(ctx.config()) && !ctx.clientNode()) {
        if (ctx.clientNode()) {
            U.warn(log, "Persistent Store is not supported on client nodes (Persistent Store's" + " configuration will be ignored).");
        }
        dbMgr = new GridCacheDatabaseSharedManager(ctx);
        pageStoreMgr = new FilePageStoreManager(ctx);
        if (ctx.config().getDataStorageConfiguration().getWalMode() == WALMode.FSYNC && !walFsyncWithDedicatedWorker)
            walMgr = new FsyncModeFileWriteAheadLogManager(ctx);
        else
            walMgr = new FileWriteAheadLogManager(ctx);
    } else
        dbMgr = new IgniteCacheDatabaseSharedManager();
    WalStateManager walStateMgr = new WalStateManager(ctx);
    IgniteCacheSnapshotManager snpMgr = ctx.plugins().createComponent(IgniteCacheSnapshotManager.class);
    if (snpMgr == null)
        snpMgr = new IgniteCacheSnapshotManager();
    GridCacheIoManager ioMgr = new GridCacheIoManager();
    CacheAffinitySharedManager topMgr = new CacheAffinitySharedManager();
    GridCacheSharedTtlCleanupManager ttl = new GridCacheSharedTtlCleanupManager();
    CacheJtaManagerAdapter jta = JTA.createOptional();
    return new GridCacheSharedContext(kernalCtx, tm, verMgr, mvccMgr, pageStoreMgr, walMgr, walStateMgr, dbMgr, snpMgr, depMgr, exchMgr, topMgr, ioMgr, ttl, jta, storeSesLsnrs);
}

Example 9 with FileWriteAheadLogManager

use of org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager in project ignite by apache.

the class IgniteWalHistoryReservationsTest method testReservedOnExchange.

/**
 * @throws Exception If failed.
 */
public void testReservedOnExchange() throws Exception {
    System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
    final int entryCnt = 10_000;
    final int initGridCnt = 4;
    final IgniteEx ig0 = (IgniteEx) startGrids(initGridCnt + 1);
    ig0.active(true);
    stopGrid(initGridCnt);
    Assert.assertEquals(5, ig0.context().state().clusterState().baselineTopology().consistentIds().size());
    long start = U.currentTimeMillis();
    log.warning("Start loading");
    try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")) {
        for (int k = 0; k < entryCnt; k++) {
            st.addData(k, k);
            printProgress(k);
        }
    }
    log.warning("Finish loading time:" + (U.currentTimeMillis() - start));
    forceCheckpoint();
    start = U.currentTimeMillis();
    log.warning("Start loading");
    try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")) {
        st.allowOverwrite(true);
        for (int k = 0; k < entryCnt; k++) {
            st.addData(k, k * 2);
            printProgress(k);
        }
    }
    log.warning("Finish loading time:" + (U.currentTimeMillis() - start));
    forceCheckpoint();
    start = U.currentTimeMillis();
    log.warning("Start loading");
    try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")) {
        st.allowOverwrite(true);
        for (int k = 0; k < entryCnt; k++) {
            st.addData(k, k);
            printProgress(k);
        }
    }
    log.warning("Finish loading time:" + (U.currentTimeMillis() - start));
    forceCheckpoint();
    Lock lock = ig0.cache("cache1").lock(0);
    lock.lock();
    try {
        GridTestUtils.runAsync(new Runnable() {

            @Override
            public void run() {
                try {
                    startGrid(initGridCnt);
                } catch (Exception e) {
                    fail(e.getMessage());
                }
            }
        });
        boolean reserved = GridTestUtils.waitForCondition(new GridAbsPredicate() {

            @Override
            public boolean apply() {
                for (int g = 0; g < initGridCnt; g++) {
                    IgniteEx ig = grid(g);
                    FileWriteAheadLogManager wal = (FileWriteAheadLogManager) ig.context().cache().context().wal();
                    Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
                    synchronized (reservationStorage) {
                        Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
                        if (reserved.isEmpty())
                            return false;
                    }
                }
                return true;
            }
        }, 10_000);
        assert reserved;
    } finally {
        lock.unlock();
    }
    boolean released = GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            for (int g = 0; g < initGridCnt; g++) {
                IgniteEx ig = grid(g);
                FileWriteAheadLogManager wal = (FileWriteAheadLogManager) ig.context().cache().context().wal();
                Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
                synchronized (reservationStorage) {
                    Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
                    if (!reserved.isEmpty())
                        return false;
                }
            }
            return true;
        }
    }, 10_000);
    assert released;
}