use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.
the class ProgressWatchdog method tick.
/**
* Regular method printing statistics to out and to log. Checks gaps in progress.
*/
private void tick() {
long elapsedMs = U.currentTimeMillis() - msStart;
final long totalCnt = overallRecordsProcessed.longValue();
long elapsedMsFromPrevTick = elapsedMs - prevMsElapsed.getAndSet(elapsedMs);
if (elapsedMsFromPrevTick == 0)
return;
final long currPutPerSec = ((totalCnt - prevRecordsCnt.getAndSet(totalCnt)) * 1000) / elapsedMsFromPrevTick;
final long averagePutPerSec = totalCnt * 1000 / elapsedMs;
boolean slowProgress = currPutPerSec < averagePutPerSec / 10 && !stopping;
final String fileNameWithDump = slowProgress ? reactNoProgress() : "";
DataStorageConfiguration dsCfg = ignite.configuration().getDataStorageConfiguration();
String defRegName = dsCfg.getDefaultDataRegionConfiguration().getName();
long dirtyPages = -1;
for (DataRegionMetrics m : ignite.dataRegionMetrics()) if (m.getName().equals(defRegName))
dirtyPages = m.getDirtyPages();
GridCacheSharedContext<Object, Object> cacheSctx = null;
PageMemoryImpl pageMemory = null;
try {
cacheSctx = ((IgniteEx) ignite).context().cache().context();
pageMemory = (PageMemoryImpl) cacheSctx.database().dataRegion(defRegName).pageMemory();
} catch (IgniteCheckedException e) {
e.printStackTrace();
}
long cpBufPages = 0;
GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (cacheSctx.database());
AtomicInteger wrPageCntr = db.writtenPagesCounter();
long cpWrittenPages = wrPageCntr == null ? 0 : wrPageCntr.get();
AtomicInteger syncedPagesCntr = db.syncedPagesCounter();
int cpSyncedPages = syncedPagesCntr == null ? 0 : syncedPagesCntr.get();
AtomicInteger evictedPagesCntr = db.evictedPagesCntr();
int cpEvictedPages = evictedPagesCntr == null ? 0 : evictedPagesCntr.get();
int pageSize = pageMemory == null ? 0 : pageMemory.pageSize();
String cpWriteSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpWrittenPages, prevCpWrittenPages) * pageSize);
String cpSyncSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpSyncedPages, prevCpSyncedPages) * pageSize);
String walSpeed = "";
long throttleParkTimeNanos = 0;
double curDirtyRatio = 0.0;
String targetDirtyRatioStr = "";
double closeToThrottle = 0.0;
long idx = -1;
long lastArchIdx = -1;
int walArchiveSegments = 0;
long walWorkSegments = 0;
long markDirtySpeed = 0;
long cpWriteSpeedInPages = 0;
long estWrAllSpeed = 0;
try {
if (pageMemory != null) {
cpBufPages = pageMemory.checkpointBufferPagesCount();
PagesWriteSpeedBasedThrottle throttle = U.field(pageMemory, "writeThrottle");
if (throttle != null) {
curDirtyRatio = throttle.getCurrDirtyRatio();
double targetDirtyRatio = throttle.getTargetDirtyRatio();
targetDirtyRatioStr = targetDirtyRatio < 0 ? "" : formatDbl(targetDirtyRatio);
closeToThrottle = throttle.throttleWeight();
throttleParkTimeNanos = throttle.throttleParkTime();
markDirtySpeed = throttle.getMarkDirtySpeed();
cpWriteSpeedInPages = throttle.getCpWriteSpeed();
estWrAllSpeed = throttle.getLastEstimatedSpeedForMarkAll();
if (estWrAllSpeed > 99_999)
estWrAllSpeed = 99_999;
}
}
FileWriteAheadLogManager wal = (FileWriteAheadLogManager) cacheSctx.wal();
idx = 0;
lastArchIdx = 0;
walArchiveSegments = wal.walArchiveSegments();
walWorkSegments = idx - lastArchIdx;
/* // uncomment when currentWritePointer is available
FileWALPointer ptr = wal.currentWritePointer();
FileWALPointer prevWalPtr = this.prevWalPtrRef.getAndSet(ptr);
if (prevWalPtr != null) {
long idxDiff = ptr.index() - prevWalPtr.index();
long offDiff = ptr.fileOffset() - prevWalPtr.fileOffset();
long bytesDiff = idxDiff * maxWalSegmentSize + offDiff;
long bytesPerSec = (bytesDiff * 1000) / elapsedMsFromPrevTick;
walSpeed = getMBytesPrintable(bytesPerSec);
} else
walSpeed = "0";
*/
walSpeed = "0";
} catch (Exception e) {
X.error(e.getClass().getSimpleName() + ":" + e.getMessage());
}
long elapsedSecs = elapsedMs / 1000;
X.println(" >> " + operation + " done: " + totalCnt + "/" + elapsedSecs + "s, " + "Cur. " + operation + " " + currPutPerSec + " recs/sec " + "cpWriteSpeed=" + cpWriteSpeed + " " + "cpSyncSpeed=" + cpSyncSpeed + " " + "walSpeed= " + walSpeed + " " + "walWorkSeg.=" + walWorkSegments + " " + "markDirtySpeed=" + markDirtySpeed + " " + "Avg. " + operation + " " + averagePutPerSec + " recs/sec, " + "dirtyP=" + dirtyPages + ", " + "cpWrittenP.=" + cpWrittenPages + ", " + "cpBufP.=" + cpBufPages + " " + "threshold=" + targetDirtyRatioStr + " " + "walIdx=" + idx + " " + "archWalIdx=" + lastArchIdx + " " + "walArchiveSegments=" + walArchiveSegments + " " + fileNameWithDump);
line(elapsedSecs, currPutPerSec, walSpeed, cpWriteSpeed, cpSyncSpeed, walWorkSegments, throttleParkTimeNanos, formatDbl(curDirtyRatio), targetDirtyRatioStr, formatDbl(closeToThrottle), markDirtySpeed, cpWriteSpeedInPages, estWrAllSpeed, averagePutPerSec, dirtyPages, cpWrittenPages, cpSyncedPages, cpEvictedPages, idx, lastArchIdx, walArchiveSegments);
}
use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.
the class FileWriteAheadLogManager method start0.
/**
* {@inheritDoc}
*/
@Override
public void start0() throws IgniteCheckedException {
if (!cctx.kernalContext().clientNode()) {
final PdsFolderSettings resolveFolders = cctx.kernalContext().pdsFolderResolver().resolveFolders();
checkWalConfiguration();
walWorkDir = initDirectory(dsCfg.getWalPath(), DataStorageConfiguration.DFLT_WAL_PATH, resolveFolders.folderName(), "write ahead log work directory");
walArchiveDir = initDirectory(dsCfg.getWalArchivePath(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, resolveFolders.folderName(), "write ahead log archive directory");
serializer = new RecordSerializerFactoryImpl(cctx).createSerializer(serializerVer);
GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) cctx.database();
metrics = dbMgr.persistentStoreMetricsImpl();
checkOrPrepareFiles();
IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
long lastAbsArchivedIdx = tup == null ? -1 : tup.get2();
if (isArchiverEnabled())
archiver = new FileArchiver(lastAbsArchivedIdx);
else
archiver = null;
if (lastAbsArchivedIdx > 0)
archivedMonitor.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
if (dsCfg.isWalCompactionEnabled()) {
compressor = new FileCompressor();
decompressor = new FileDecompressor();
if (!cctx.kernalContext().clientNode())
decompressor.start();
}
if (mode != WALMode.NONE) {
if (log.isInfoEnabled())
log.info("Started write-ahead log manager [mode=" + mode + ']');
} else
U.quietAndWarn(log, "Started write-ahead log manager in NONE mode, persisted data may be lost in " + "a case of unexpected node failure. Make sure to deactivate the cluster before shutdown.");
}
}
use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.
the class FsyncModeFileWriteAheadLogManager method start0.
/**
* {@inheritDoc}
*/
@Override
public void start0() throws IgniteCheckedException {
if (!cctx.kernalContext().clientNode()) {
final PdsFolderSettings resolveFolders = cctx.kernalContext().pdsFolderResolver().resolveFolders();
checkWalConfiguration();
walWorkDir = initDirectory(dsCfg.getWalPath(), DataStorageConfiguration.DFLT_WAL_PATH, resolveFolders.folderName(), "write ahead log work directory");
walArchiveDir = initDirectory(dsCfg.getWalArchivePath(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, resolveFolders.folderName(), "write ahead log archive directory");
serializer = new RecordSerializerFactoryImpl(cctx).createSerializer(serializerVersion);
GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) cctx.database();
metrics = dbMgr.persistentStoreMetricsImpl();
checkOrPrepareFiles();
IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
archiver = new FileArchiver(tup == null ? -1 : tup.get2());
if (dsCfg.isWalCompactionEnabled()) {
compressor = new FileCompressor();
decompressor = new FileDecompressor();
}
if (mode != WALMode.NONE) {
if (log.isInfoEnabled())
log.info("Started write-ahead log manager [mode=" + mode + ']');
} else
U.quietAndWarn(log, "Started write-ahead log manager in NONE mode, persisted data may be lost in " + "a case of unexpected node failure. Make sure to deactivate the cluster before shutdown.");
}
}
use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.
the class GridCommonAbstractTest method forceCheckpoint.
/**
* Forces checkpoint on all specified nodes.
*
* @param nodes Nodes to force checkpoint on them.
* @throws IgniteCheckedException If checkpoint was failed.
*/
protected void forceCheckpoint(Collection<Ignite> nodes) throws IgniteCheckedException {
for (Ignite ignite : nodes) {
if (ignite.cluster().localNode().isClient())
continue;
GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ((IgniteEx) ignite).context().cache().context().database();
dbMgr.waitForCheckpoint("test");
}
}
use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.
the class GridCacheProcessor method createSharedContext.
/**
* Creates shared context.
*
* @param kernalCtx Kernal context.
* @param storeSesLsnrs Store session listeners.
* @return Shared context.
* @throws IgniteCheckedException If failed.
*/
@SuppressWarnings("unchecked")
private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx, Collection<CacheStoreSessionListener> storeSesLsnrs) throws IgniteCheckedException {
IgniteTxManager tm = new IgniteTxManager();
GridCacheMvccManager mvccMgr = new GridCacheMvccManager();
GridCacheVersionManager verMgr = new GridCacheVersionManager();
GridCacheDeploymentManager depMgr = new GridCacheDeploymentManager();
GridCachePartitionExchangeManager exchMgr = new GridCachePartitionExchangeManager();
IgniteCacheDatabaseSharedManager dbMgr;
IgnitePageStoreManager pageStoreMgr = null;
IgniteWriteAheadLogManager walMgr = null;
if (CU.isPersistenceEnabled(ctx.config()) && !ctx.clientNode()) {
if (ctx.clientNode()) {
U.warn(log, "Persistent Store is not supported on client nodes (Persistent Store's" + " configuration will be ignored).");
}
dbMgr = new GridCacheDatabaseSharedManager(ctx);
pageStoreMgr = new FilePageStoreManager(ctx);
if (ctx.config().getDataStorageConfiguration().getWalMode() == WALMode.FSYNC && !walFsyncWithDedicatedWorker)
walMgr = new FsyncModeFileWriteAheadLogManager(ctx);
else
walMgr = new FileWriteAheadLogManager(ctx);
} else
dbMgr = new IgniteCacheDatabaseSharedManager();
WalStateManager walStateMgr = new WalStateManager(ctx);
IgniteCacheSnapshotManager snpMgr = ctx.plugins().createComponent(IgniteCacheSnapshotManager.class);
if (snpMgr == null)
snpMgr = new IgniteCacheSnapshotManager();
GridCacheIoManager ioMgr = new GridCacheIoManager();
CacheAffinitySharedManager topMgr = new CacheAffinitySharedManager();
GridCacheSharedTtlCleanupManager ttl = new GridCacheSharedTtlCleanupManager();
CacheJtaManagerAdapter jta = JTA.createOptional();
return new GridCacheSharedContext(kernalCtx, tm, verMgr, mvccMgr, pageStoreMgr, walMgr, walStateMgr, dbMgr, snpMgr, depMgr, exchMgr, topMgr, ioMgr, ttl, jta, storeSesLsnrs);
}
Aggregations