Examples with GridCacheDatabaseSharedManager org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager used on opensource projects

Example 56 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class RestorePartitionStateTest method testLogTopPartitions.

/**
 * Check that the progress of restoring partitions with the top partitions is displayed in the log.
 *
 * @throws Exception If fail.
 */
@Test
public void testLogTopPartitions() throws Exception {
    IgniteEx n = startGrid(0);
    n.cluster().state(ClusterState.ACTIVE);
    awaitPartitionMapExchange();
    ((GridCacheDatabaseSharedManager) n.context().cache().context().database()).enableCheckpoints(false).get(getTestTimeout());
    for (IgniteInternalCache cache : n.context().cache().caches()) {
        for (int i = 0; i < 1_000; i++) cache.put(i, cache.name() + i);
    }
    stopAllGrids();
    awaitPartitionMapExchange();
    LogListener startPartRestore = LogListener.matches(logStr -> {
        if (logStr.startsWith("Started restoring partition state [grp=")) {
            try {
                U.sleep(15);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteException(e);
            }
            return true;
        }
        return false;
    }).build();
    LogListener progressPartRestore = LogListener.matches("Restore partitions state progress").andMatches("topProcessedPartitions").build();
    LogListener finishPartRestore = LogListener.matches("Finished restoring partition state for local groups").andMatches("topProcessedPartitions").build();
    TIMEOUT_OUTPUT_RESTORE_PARTITION_STATE_PROGRESS = 150L;
    setRootLoggerDebugLevel();
    startGrid(0, (UnaryOperator<IgniteConfiguration>) cfg -> {
        cfg.setGridLogger(new ListeningTestLogger(log, startPartRestore, progressPartRestore, finishPartRestore));
        return cfg;
    });
    assertTrue(startPartRestore.check());
    assertTrue(progressPartRestore.check());
    assertTrue(finishPartRestore.check());
}

Example 57 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class CheckpointFreeListTest method testFreeListUnderLoadMultipleCheckpoints.

/**
 * Test checks that free-list works and pages cache flushes correctly under the high concurrent load.
 */
@Test
public void testFreeListUnderLoadMultipleCheckpoints() throws Throwable {
    IgniteEx ignite = startGrid(0);
    ignite.cluster().active(true);
    int minValSize = 64;
    int maxValSize = 128;
    int valsCnt = maxValSize - minValSize;
    int keysCnt = 1_000;
    byte[][] vals = new byte[valsCnt][];
    for (int i = 0; i < valsCnt; i++) vals[i] = new byte[minValSize + i];
    IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME).setAffinity(// Maximize contention per partition.
    new RendezvousAffinityFunction().setPartitions(2)).setAtomicityMode(CacheAtomicityMode.ATOMIC));
    AtomicBoolean done = new AtomicBoolean();
    AtomicReference<Throwable> error = new AtomicReference<>();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) ignite.context().cache().context().database();
    AtomicLong pageListCacheLimitHolder = db.pageListCacheLimitHolder(ignite.context().cache().cache(DEFAULT_CACHE_NAME).context().dataRegion());
    long initPageListCacheLimit = pageListCacheLimitHolder.get();
    // Add listener after cache is started, so this listener will be triggered after listener for cache.
    db.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
            // Need to wait for condition here, since checkpointer can store free-list metadata asynchronously.
            if (!waitForCondition(() -> initPageListCacheLimit == pageListCacheLimitHolder.get(), 1_000L)) {
                IgniteCheckedException e = new IgniteCheckedException("Page list cache limit doesn't restored " + "correctly [init=" + initPageListCacheLimit + ", cur=" + pageListCacheLimitHolder.get() + ']');
                error.set(e);
                throw e;
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) {
        // No-op.
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) {
        // No-op.
        }
    });
    IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(() -> {
        Random rnd = new Random();
        try {
            while (!done.get()) {
                int key = rnd.nextInt(keysCnt);
                byte[] val = vals[rnd.nextInt(valsCnt)];
                // Put with changed value size - worst case for free list, since row will be removed first and
                // then inserted again.
                cache.put(key, val);
            }
        } catch (Throwable t) {
            error.set(t);
        }
    }, 20, "cache-put");
    for (int i = 0; i < SF.applyLB(10, 2); i++) {
        if (error.get() != null)
            break;
        forceCheckpoint(ignite);
        doSleep(1_000L);
    }
    done.set(true);
    fut.get();
    stopAllGrids();
    if (error.get() != null)
        throw error.get();
}

Example 58 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteCheckpointDirtyPagesForLowLoadTest method testManyCachesAndNotManyPuts.

/**
 * @throws Exception if failed.
 */
@Test
public void testManyCachesAndNotManyPuts() throws Exception {
    try {
        IgniteEx ignite = startGrid(0);
        ignite.active(true);
        log.info("Saving initial data to caches");
        for (int g = 0; g < GROUPS; g++) {
            for (int c = 0; c < CACHES_IN_GRP; c++) {
                ignite.cache("dummyCache" + c + "." + g).putAll(new TreeMap<Long, Long>() {

                    {
                        for (int j = 0; j < PARTS; j++) {
                            // to fill each partition cache with at least 1 element
                            put((long) j, (long) j);
                        }
                    }
                });
            }
        }
        GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (ignite.context().cache().context().database());
        Collection<Integer> pageCntObserved = new ArrayList<>();
        boolean checkpointWithLowNumOfPagesFound = false;
        for (int i = 0; i < 20; i++) {
            Random random = new Random();
            // touch some entry
            int d = random.nextInt(PARTS) + PARTS;
            int cIdx = random.nextInt(CACHES_IN_GRP);
            int gIdx = random.nextInt(GROUPS);
            String fullname = "dummyCache" + cIdx + "." + gIdx;
            ignite.cache(fullname).put(d, d);
            if (log.isInfoEnabled())
                log.info("Put to cache [" + fullname + "] value " + d);
            long start = System.nanoTime();
            try {
                final int cpTimeout = 25000;
                db.wakeupForCheckpoint("").get(cpTimeout, TimeUnit.MILLISECONDS);
            } catch (IgniteFutureTimeoutCheckedException ignored) {
                long msPassed = U.millisSinceNanos(start);
                log.error("Timeout during waiting for checkpoint to start:" + " [" + msPassed + "] but checkpoint is not running");
                continue;
            }
            final int timeout = 5000;
            int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db, timeout);
            if (currCpPages < 0) {
                log.error("Timeout during waiting for checkpoint counter to be updated");
                continue;
            }
            pageCntObserved.add(currCpPages);
            log.info("Current CP pages: " + currCpPages);
            if (currCpPages < PARTS * GROUPS) {
                // reasonable number of pages in CP
                checkpointWithLowNumOfPagesFound = true;
                break;
            }
        }
        stopGrid(0);
        assertTrue("All checkpoints mark too much pages: " + pageCntObserved, checkpointWithLowNumOfPagesFound);
    } finally {
        stopAllGrids();
    }
}

Example 59 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class LightweightCheckpointTest method testLightCheckpointAbleToStoreOnlyGivenDataRegion.

/**
 * 1. Start the one node with disabled WAL and with two caches.
 * 2. Disable default checkpoint.
 * 3. Create light checkpoint for one cache and configure checkpoint listener for it.
 * 4. Fill the both caches.
 * 5. Trigger the light checkpoint and wait for the finish.
 * 6. Stop the node and start it again.
 * 7. Expected: Cache which was checkpointed would have the all data meanwhile second cache would be empty.
 *
 * @throws Exception if fail.
 */
@Test
public void testLightCheckpointAbleToStoreOnlyGivenDataRegion() throws Exception {
    // given: One started node with default cache and cache which won't be checkpointed.
    IgniteEx ignite0 = startGrid(0);
    ignite0.cluster().active(true);
    IgniteCache<Integer, Object> checkpointedCache = ignite0.cache(DEFAULT_CACHE_NAME);
    IgniteCache<Integer, Object> notCheckpointedCache = ignite0.cache(NOT_CHECKPOINTED_CACHE);
    GridKernalContext context = ignite0.context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) (context.cache().context().database());
    waitForCondition(() -> !db.getCheckpointer().currentProgress().inProgress(), 10_000);
    // and: disable the default checkpoint.
    db.enableCheckpoints(false);
    DataRegion regionForCheckpoint = db.dataRegion(DFLT_DATA_REG_DEFAULT_NAME);
    // and: Create light checkpoint with only one region.
    LightweightCheckpointManager lightweightCheckpointManager = new LightweightCheckpointManager(context::log, context.igniteInstanceName(), "light-test-checkpoint", context.workersRegistry(), context.config().getDataStorageConfiguration(), () -> Arrays.asList(regionForCheckpoint), grpId -> getPageMemoryForCacheGroup(grpId, db, context), PageMemoryImpl.ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY, context.cache().context().snapshot(), db.persistentStoreMetricsImpl(), context.longJvmPauseDetector(), context.failure(), context.cache());
    // and: Add checkpoint listener for DEFAULT_CACHE in order of storing the meta pages.
    lightweightCheckpointManager.addCheckpointListener((CheckpointListener) context.cache().cacheGroup(groupIdForCache(ignite0, DEFAULT_CACHE_NAME)).offheap(), regionForCheckpoint);
    lightweightCheckpointManager.start();
    // when: Fill the caches
    for (int j = 0; j < 1024; j++) {
        checkpointedCache.put(j, j);
        notCheckpointedCache.put(j, j);
    }
    // and: Trigger and wait for the checkpoint.
    lightweightCheckpointManager.forceCheckpoint("test", null).futureFor(CheckpointState.FINISHED).get();
    // and: Stop and start node.
    stopAllGrids();
    ignite0 = startGrid(0);
    ignite0.cluster().active(true);
    checkpointedCache = ignite0.cache(DEFAULT_CACHE_NAME);
    notCheckpointedCache = ignite0.cache(NOT_CHECKPOINTED_CACHE);
    // then: Checkpointed cache should have all data meanwhile uncheckpointed cache should be empty.
    for (int j = 1; j < 1024; j++) {
        assertEquals(j, checkpointedCache.get(j));
        assertNull(notCheckpointedCache.get(j));
    }
    GridCacheDatabaseSharedManager db2 = (GridCacheDatabaseSharedManager) (ignite0.context().cache().context().database());
    waitForCondition(() -> !db2.getCheckpointer().currentProgress().inProgress(), 10_000);
    String nodeFolderName = ignite0.context().pdsFolderResolver().resolveFolders().folderName();
    File cpMarkersDir = Paths.get(U.defaultWorkDirectory(), "db", nodeFolderName, "cp").toFile();
    // then: Expected only two pairs checkpoint markers - both from the start of node.
    assertEquals(4, cpMarkersDir.listFiles().length);
}

Example 60 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteWalIteratorSwitchSegmentTest method initiate.

/**
 * Initiate WAL manager.
 *
 * @param serVer WAL serializer version.
 * @param workDir Work directory path.
 * @return Tuple of WAL manager and WAL record serializer.
 * @throws IgniteCheckedException If some think failed.
 */
private T2<IgniteWriteAheadLogManager, RecordSerializer> initiate(int serVer, String workDir) throws IgniteCheckedException {
    GridKernalContext kctx = new StandaloneGridKernalContext(log, null, null) {

        @Override
        protected IgniteConfiguration prepareIgniteConfiguration() {
            IgniteConfiguration cfg = super.prepareIgniteConfiguration();
            cfg.setDataStorageConfiguration(new DataStorageConfiguration().setWalSegmentSize(SEGMENT_SIZE).setWalRecordIteratorBufferSize(SEGMENT_SIZE / 2).setWalMode(WALMode.FSYNC).setWalPath(workDir + WORK_SUB_DIR).setWalArchivePath(workDir + ARCHIVE_SUB_DIR).setFileIOFactory(new RandomAccessFileIOFactory()));
            cfg.setEventStorageSpi(new NoopEventStorageSpi());
            return cfg;
        }

        @Override
        public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
            return new GridInternalSubscriptionProcessor(this);
        }

        @Override
        public GridEventStorageManager event() {
            return new GridEventStorageManager(this);
        }
    };
    IgniteWriteAheadLogManager walMgr = new FileWriteAheadLogManager(kctx);
    GridTestUtils.setFieldValue(walMgr, "serializerVer", serVer);
    GridCacheSharedContext<?, ?> ctx = new GridCacheSharedContext<>(kctx, null, null, null, null, walMgr, new WalStateManager(kctx), new GridCacheDatabaseSharedManager(kctx), null, null, null, null, null, new GridCacheIoManager(), null, null, null, null, null, null, null);
    walMgr.start(ctx);
    walMgr.onActivate(kctx);
    walMgr.resumeLogging(null);
    RecordSerializer recordSerializer = new RecordSerializerFactoryImpl(ctx).createSerializer(walMgr.serializerVersion());
    return new T2<>(walMgr, recordSerializer);
}