Examples with DataStorageConfiguration org.apache.ignite.configuration.DataStorageConfiguration used on opensource projects

Example 56 with DataStorageConfiguration

use of org.apache.ignite.configuration.DataStorageConfiguration in project ignite by apache.

the class IgniteCacheLargeValueExpireTest method getConfiguration.

/**
 * {@inheritDoc}
 */
@Override
protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
    IgniteConfiguration cfg = super.getConfiguration(gridName);
    ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
    DataStorageConfiguration dbCfg = new DataStorageConfiguration();
    dbCfg.setPageSize(1024);
    cfg.setDataStorageConfiguration(dbCfg);
    return cfg;
}

Example 57 with DataStorageConfiguration

use of org.apache.ignite.configuration.DataStorageConfiguration in project ignite by apache.

the class IgnitePdsCacheRebalancingAbstractTest method getConfiguration.

/**
 * {@inheritDoc}
 */
@Override
protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
    IgniteConfiguration cfg = super.getConfiguration(gridName);
    cfg.setConsistentId(gridName);
    CacheConfiguration ccfg1 = cacheConfiguration(cacheName).setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE).setBackups(2).setRebalanceMode(CacheRebalanceMode.ASYNC).setIndexedTypes(Integer.class, Integer.class).setAffinity(new RendezvousAffinityFunction(false, 32)).setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    CacheConfiguration ccfg2 = cacheConfiguration("indexed");
    ccfg2.setBackups(1);
    ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    QueryEntity qryEntity = new QueryEntity(Integer.class.getName(), TestValue.class.getName());
    LinkedHashMap<String, String> fields = new LinkedHashMap<>();
    fields.put("v1", Integer.class.getName());
    fields.put("v2", Integer.class.getName());
    qryEntity.setFields(fields);
    QueryIndex qryIdx = new QueryIndex("v1", true);
    qryEntity.setIndexes(Collections.singleton(qryIdx));
    ccfg2.setQueryEntities(Collections.singleton(qryEntity));
    // Do not start filtered cache on coordinator.
    if (gridName.endsWith("0")) {
        cfg.setCacheConfiguration(ccfg1, ccfg2);
    } else {
        CacheConfiguration ccfg3 = cacheConfiguration("filtered");
        ccfg3.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
        ccfg3.setBackups(1);
        ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        ccfg3.setNodeFilter(new CoordinatorNodeFilter());
        cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3);
    }
    DataStorageConfiguration memCfg = new DataStorageConfiguration();
    memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
    memCfg.setPageSize(1024);
    memCfg.setWalMode(WALMode.LOG_ONLY);
    DataRegionConfiguration memPlcCfg = new DataRegionConfiguration();
    memPlcCfg.setName("dfltDataRegion");
    memPlcCfg.setMaxSize(150 * 1024 * 1024);
    memPlcCfg.setInitialSize(100 * 1024 * 1024);
    memPlcCfg.setSwapPath("work/swap");
    memPlcCfg.setPersistenceEnabled(true);
    memCfg.setDefaultDataRegionConfiguration(memPlcCfg);
    cfg.setDataStorageConfiguration(memCfg);
    cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
    return cfg;
}

Example 58 with DataStorageConfiguration

use of org.apache.ignite.configuration.DataStorageConfiguration in project ignite by apache.

the class IgnitePdsPageReplacementTest method createDbConfig.

/**
 * @return DB config.
 */
private DataStorageConfiguration createDbConfig() {
    final DataStorageConfiguration memCfg = new DataStorageConfiguration();
    DataRegionConfiguration memPlcCfg = new DataRegionConfiguration();
    memPlcCfg.setInitialSize(MEMORY_LIMIT);
    memPlcCfg.setMaxSize(MEMORY_LIMIT);
    memPlcCfg.setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU);
    memPlcCfg.setName("dfltDataRegion");
    memPlcCfg.setPersistenceEnabled(true);
    memCfg.setPageSize(PAGE_SIZE);
    memCfg.setConcurrencyLevel(NUMBER_OF_SEGMENTS);
    memCfg.setDefaultDataRegionConfiguration(memPlcCfg);
    memCfg.setWalMode(WALMode.LOG_ONLY);
    return memCfg;
}

Example 59 with DataStorageConfiguration

use of org.apache.ignite.configuration.DataStorageConfiguration in project ignite by apache.

the class PageEvictionAbstractTest method getConfiguration.

/**
 * {@inheritDoc}
 */
@Override
protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
    IgniteConfiguration cfg = super.getConfiguration(gridName);
    ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
    DataStorageConfiguration dbCfg = new DataStorageConfiguration();
    DataRegionConfiguration plc = new DataRegionConfiguration();
    // This will test additional segment allocation.
    plc.setInitialSize(SIZE / 2);
    plc.setMaxSize(SIZE);
    plc.setEmptyPagesPoolSize(EMPTY_PAGES_POOL_SIZE);
    plc.setEvictionThreshold(EVICTION_THRESHOLD);
    plc.setName(DEFAULT_POLICY_NAME);
    dbCfg.setDefaultDataRegionConfiguration(plc);
    dbCfg.setPageSize(PAGE_SIZE);
    cfg.setDataStorageConfiguration(dbCfg);
    return cfg;
}

Example 60 with DataStorageConfiguration

use of org.apache.ignite.configuration.DataStorageConfiguration in project ignite by apache.

the class ProgressWatchdog method tick.

/**
 * Regular method printing statistics to out and to log. Checks gaps in progress.
 */
private void tick() {
    long elapsedMs = U.currentTimeMillis() - msStart;
    final long totalCnt = overallRecordsProcessed.longValue();
    long elapsedMsFromPrevTick = elapsedMs - prevMsElapsed.getAndSet(elapsedMs);
    if (elapsedMsFromPrevTick == 0)
        return;
    final long currPutPerSec = ((totalCnt - prevRecordsCnt.getAndSet(totalCnt)) * 1000) / elapsedMsFromPrevTick;
    final long averagePutPerSec = totalCnt * 1000 / elapsedMs;
    boolean slowProgress = currPutPerSec < averagePutPerSec / 10 && !stopping;
    final String fileNameWithDump = slowProgress ? reactNoProgress() : "";
    DataStorageConfiguration dsCfg = ignite.configuration().getDataStorageConfiguration();
    String defRegName = dsCfg.getDefaultDataRegionConfiguration().getName();
    long dirtyPages = -1;
    for (DataRegionMetrics m : ignite.dataRegionMetrics()) if (m.getName().equals(defRegName))
        dirtyPages = m.getDirtyPages();
    GridCacheSharedContext<Object, Object> cacheSctx = null;
    PageMemoryImpl pageMemory = null;
    try {
        cacheSctx = ((IgniteEx) ignite).context().cache().context();
        pageMemory = (PageMemoryImpl) cacheSctx.database().dataRegion(defRegName).pageMemory();
    } catch (IgniteCheckedException e) {
        e.printStackTrace();
    }
    long cpBufPages = 0;
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (cacheSctx.database());
    AtomicInteger wrPageCntr = db.writtenPagesCounter();
    long cpWrittenPages = wrPageCntr == null ? 0 : wrPageCntr.get();
    AtomicInteger syncedPagesCntr = db.syncedPagesCounter();
    int cpSyncedPages = syncedPagesCntr == null ? 0 : syncedPagesCntr.get();
    AtomicInteger evictedPagesCntr = db.evictedPagesCntr();
    int cpEvictedPages = evictedPagesCntr == null ? 0 : evictedPagesCntr.get();
    int pageSize = pageMemory == null ? 0 : pageMemory.pageSize();
    String cpWriteSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpWrittenPages, prevCpWrittenPages) * pageSize);
    String cpSyncSpeed = getMBytesPrintable(detectDelta(elapsedMsFromPrevTick, cpSyncedPages, prevCpSyncedPages) * pageSize);
    String walSpeed = "";
    long throttleParkTimeNanos = 0;
    double curDirtyRatio = 0.0;
    String targetDirtyRatioStr = "";
    double closeToThrottle = 0.0;
    long idx = -1;
    long lastArchIdx = -1;
    int walArchiveSegments = 0;
    long walWorkSegments = 0;
    long markDirtySpeed = 0;
    long cpWriteSpeedInPages = 0;
    long estWrAllSpeed = 0;
    try {
        if (pageMemory != null) {
            cpBufPages = pageMemory.checkpointBufferPagesCount();
            PagesWriteSpeedBasedThrottle throttle = U.field(pageMemory, "writeThrottle");
            if (throttle != null) {
                curDirtyRatio = throttle.getCurrDirtyRatio();
                double targetDirtyRatio = throttle.getTargetDirtyRatio();
                targetDirtyRatioStr = targetDirtyRatio < 0 ? "" : formatDbl(targetDirtyRatio);
                closeToThrottle = throttle.throttleWeight();
                throttleParkTimeNanos = throttle.throttleParkTime();
                markDirtySpeed = throttle.getMarkDirtySpeed();
                cpWriteSpeedInPages = throttle.getCpWriteSpeed();
                estWrAllSpeed = throttle.getLastEstimatedSpeedForMarkAll();
                if (estWrAllSpeed > 99_999)
                    estWrAllSpeed = 99_999;
            }
        }
        FileWriteAheadLogManager wal = (FileWriteAheadLogManager) cacheSctx.wal();
        idx = 0;
        lastArchIdx = 0;
        walArchiveSegments = wal.walArchiveSegments();
        walWorkSegments = idx - lastArchIdx;
        /* // uncomment when currentWritePointer is available
             FileWALPointer ptr = wal.currentWritePointer();
               FileWALPointer prevWalPtr = this.prevWalPtrRef.getAndSet(ptr);

               if (prevWalPtr != null) {
                   long idxDiff = ptr.index() - prevWalPtr.index();
                   long offDiff = ptr.fileOffset() - prevWalPtr.fileOffset();
                   long bytesDiff = idxDiff * maxWalSegmentSize + offDiff;

                   long bytesPerSec = (bytesDiff * 1000) / elapsedMsFromPrevTick;

                   walSpeed = getMBytesPrintable(bytesPerSec);
               } else
                   walSpeed = "0";
             */
        walSpeed = "0";
    } catch (Exception e) {
        X.error(e.getClass().getSimpleName() + ":" + e.getMessage());
    }
    long elapsedSecs = elapsedMs / 1000;
    X.println(" >> " + operation + " done: " + totalCnt + "/" + elapsedSecs + "s, " + "Cur. " + operation + " " + currPutPerSec + " recs/sec " + "cpWriteSpeed=" + cpWriteSpeed + " " + "cpSyncSpeed=" + cpSyncSpeed + " " + "walSpeed= " + walSpeed + " " + "walWorkSeg.=" + walWorkSegments + " " + "markDirtySpeed=" + markDirtySpeed + " " + "Avg. " + operation + " " + averagePutPerSec + " recs/sec, " + "dirtyP=" + dirtyPages + ", " + "cpWrittenP.=" + cpWrittenPages + ", " + "cpBufP.=" + cpBufPages + " " + "threshold=" + targetDirtyRatioStr + " " + "walIdx=" + idx + " " + "archWalIdx=" + lastArchIdx + " " + "walArchiveSegments=" + walArchiveSegments + " " + fileNameWithDump);
    line(elapsedSecs, currPutPerSec, walSpeed, cpWriteSpeed, cpSyncSpeed, walWorkSegments, throttleParkTimeNanos, formatDbl(curDirtyRatio), targetDirtyRatioStr, formatDbl(closeToThrottle), markDirtySpeed, cpWriteSpeedInPages, estWrAllSpeed, averagePutPerSec, dirtyPages, cpWrittenPages, cpSyncedPages, cpEvictedPages, idx, lastArchIdx, walArchiveSegments);
}