Examples with GridCacheDatabaseSharedManager org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager used on opensource projects

Example 51 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteSnapshotManagerSelfTest method testSnapshotLocalPartitionMultiCpWithLoad.

/**
 * Test that all partitions are copied successfully even after multiple checkpoints occur during
 * the long copy of cache partition files.
 *
 * Data consistency checked through a test node started right from snapshot directory and all values
 * read successes.
 *
 * @throws Exception If fails.
 */
@Test
public void testSnapshotLocalPartitionMultiCpWithLoad() throws Exception {
    int valMultiplier = 2;
    CountDownLatch slowCopy = new CountDownLatch(1);
    // Start grid node with data before each test.
    IgniteEx ig = startGridsWithCache(1, CACHE_KEYS_RANGE, key -> new Account(key, key), new CacheConfiguration<>(DEFAULT_CACHE_NAME));
    GridCacheSharedContext<?, ?> cctx = ig.context().cache().context();
    AtomicInteger cntr = new AtomicInteger();
    CountDownLatch ldrLatch = new CountDownLatch(1);
    IgniteSnapshotManager mgr = snp(ig);
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) cctx.database();
    IgniteInternalFuture<?> loadFut = GridTestUtils.runMultiThreadedAsync(() -> {
        try {
            U.await(ldrLatch);
            while (!Thread.currentThread().isInterrupted()) ig.cache(DEFAULT_CACHE_NAME).put(cntr.incrementAndGet(), new Account(cntr.incrementAndGet(), cntr.incrementAndGet()));
        } catch (IgniteInterruptedCheckedException e) {
            log.warning("Loader has been interrupted", e);
        }
    }, 5, "cache-loader-");
    // Register task but not schedule it on the checkpoint.
    SnapshotFutureTask snpFutTask = (SnapshotFutureTask) mgr.registerSnapshotTask(SNAPSHOT_NAME, cctx.localNodeId(), F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) {

        @Override
        public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
            try {
                U.await(slowCopy);
                delegate.sendPart0(part, cacheDirName, pair, length);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteException(e);
            }
        }
    });
    db.addCheckpointListener(new CheckpointListener() {

        /**
         * {@inheritDoc}
         */
        @Override
        public void beforeCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onMarkCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onCheckpointBegin(Context ctx) {
            Map<Integer, Set<Integer>> processed = GridTestUtils.getFieldValue(snpFutTask, SnapshotFutureTask.class, "processed");
            if (!processed.isEmpty())
                ldrLatch.countDown();
        }
    });
    try {
        snpFutTask.start();
        // Change data before snapshot creation which must be included into it with correct value multiplier.
        for (int i = 0; i < CACHE_KEYS_RANGE; i++) ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, valMultiplier * i));
        // Snapshot is still in the INIT state. beforeCheckpoint has been skipped
        // due to checkpoint already running and we need to schedule the next one
        // right after current will be completed.
        cctx.database().forceCheckpoint(String.format(CP_SNAPSHOT_REASON, SNAPSHOT_NAME));
        snpFutTask.started().get();
        db.forceCheckpoint("snapshot is ready to be created").futureFor(CheckpointState.MARKER_STORED_TO_DISK).get();
        // Change data after snapshot.
        for (int i = 0; i < CACHE_KEYS_RANGE; i++) ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, 3 * i));
        // Snapshot on the next checkpoint must copy page to delta file before write it to a partition.
        forceCheckpoint(ig);
        slowCopy.countDown();
        snpFutTask.get();
    } finally {
        loadFut.cancel();
    }
    // Now can stop the node and check created snapshots.
    stopGrid(0);
    cleanPersistenceDir(ig.name());
    // Start Ignite instance from snapshot directory.
    IgniteEx ig2 = startGridsFromSnapshot(1, SNAPSHOT_NAME);
    for (int i = 0; i < CACHE_KEYS_RANGE; i++) {
        assertEquals("snapshot data consistency violation [key=" + i + ']', i * valMultiplier, ((Account) ig2.cache(DEFAULT_CACHE_NAME).get(i)).balance);
    }
}

Example 52 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class IgniteSnapshotWithMetastorageTest method testMetastorageUpdateOnSnapshotFail.

/**
 * @throws Exception If fails.
 */
@Test
public void testMetastorageUpdateOnSnapshotFail() throws Exception {
    AtomicInteger keyCnt = new AtomicInteger();
    AtomicBoolean stop = new AtomicBoolean();
    Set<String> writtenKeys = new ConcurrentSkipListSet<>();
    IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE);
    IgniteInternalFuture<?> updFut = GridTestUtils.runMultiThreadedAsync(() -> {
        while (!Thread.currentThread().isInterrupted() && !stop.get()) {
            try {
                String key = SNAPSHOT_PREFIX + keyCnt.getAndIncrement();
                ignite.context().distributedMetastorage().write(key, "value");
                writtenKeys.add(key);
            } catch (IgniteCheckedException e) {
                throw new IgniteException(e);
            }
        }
    }, 3, "dms-updater");
    ((GridCacheDatabaseSharedManager) ignite.context().cache().context().database()).addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) {
            if (ctx.progress().reason().contains(SNAPSHOT_NAME)) {
                Map<String, SnapshotFutureTask> locMap = GridTestUtils.getFieldValue(snp(ignite), IgniteSnapshotManager.class, "locSnpTasks");
                // Fail the snapshot task with an exception, all metastorage keys must be successfully continued.
                locMap.get(SNAPSHOT_NAME).acceptException(new IgniteCheckedException("Test exception"));
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) {
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) {
        }
    });
    IgniteFuture<?> fut = ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
    GridTestUtils.assertThrowsAnyCause(log, fut::get, IgniteCheckedException.class, "Test exception");
    stop.set(true);
    // Load future must complete without exceptions, all metastorage keys must be written.
    updFut.get();
    Set<String> readedKeys = new TreeSet<>();
    ignite.context().distributedMetastorage().iterate(SNAPSHOT_PREFIX, (key, val) -> readedKeys.add(key));
    assertEquals("Not all metastorage keys have been written", writtenKeys, readedKeys);
}

Example 53 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class ReleaseSegmentOnHistoricalRebalanceTest method testReleaseBeforeReleaseHistoryForPreloading.

/**
 * Checks that if release the segment before {@link GridCacheDatabaseSharedManager#releaseHistoryForPreloading},
 * there will be no errors and the rebalance will be completed.
 *
 * @throws Exception If failed.
 */
@Test
public void testReleaseBeforeReleaseHistoryForPreloading() throws Exception {
    checkHistoricalRebalance(n -> {
        GridCacheDatabaseSharedManager spy = spy(dbMgr(n));
        doAnswer(m -> {
            release(n, spy.latestWalPointerReservedForPreloading());
            return m.callRealMethod();
        }).when(spy).releaseHistoryForPreloading();
        databaseManager(n, spy);
    });
}

Example 54 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class ReleaseSegmentOnHistoricalRebalanceTest method testReleaseBeforeReleaseHistoryForExchange.

/**
 * Checks that if release the segment before {@link GridCacheDatabaseSharedManager#releaseHistoryForExchange},
 * there will be no errors and the rebalance will be completed.
 *
 * @throws Exception If failed.
 */
@Test
public void testReleaseBeforeReleaseHistoryForExchange() throws Exception {
    checkHistoricalRebalance(n -> {
        GridCacheDatabaseSharedManager spy = spy(dbMgr(n));
        doAnswer(m -> {
            release(n, getFieldValue(spy, "reservedForExchange"));
            return m.callRealMethod();
        }).when(spy).releaseHistoryForExchange();
        databaseManager(n, spy);
    });
}

Example 55 with GridCacheDatabaseSharedManager

use of org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager in project ignite by apache.

the class PageMemoryTracker method start.

/**
 * Start tracking pages.
 */
synchronized void start() {
    if (!isEnabled() || started)
        return;
    pageSize = ctx.igniteConfiguration().getDataStorageConfiguration().getPageSize();
    pageMemoryMock = mockPageMemory();
    GridCacheSharedContext sharedCtx = gridCtx.cache().context();
    // Initialize one memory region for all data regions of target ignite node.
    long maxMemorySize = 0;
    for (DataRegion dataRegion : sharedCtx.database().dataRegions()) {
        if (dataRegion.pageMemory() instanceof PageMemoryImpl)
            maxMemorySize += dataRegion.config().getMaxSize();
    }
    long[] chunks = new long[] { maxMemorySize };
    memoryProvider = new UnsafeMemoryProvider(log);
    memoryProvider.initialize(chunks);
    memoryRegion = memoryProvider.nextRegion();
    GridUnsafe.setMemory(memoryRegion.address(), memoryRegion.size(), (byte) 0);
    maxPages = (int) (maxMemorySize / pageSize);
    pageSlots = new DirectMemoryPageSlot[maxPages];
    freeSlotsCnt = maxPages;
    tmpBuf1 = ByteBuffer.allocateDirect(pageSize);
    tmpBuf2 = ByteBuffer.allocateDirect(pageSize);
    if (cfg.isCheckPagesOnCheckpoint()) {
        checkpointLsnr = new CheckpointListener() {

            @Override
            public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
                if (!checkPages(false, true))
                    throw new IgniteCheckedException("Page memory is inconsistent after applying WAL delta records.");
            }

            @Override
            public void beforeCheckpointBegin(Context ctx) {
            /* No-op. */
            }

            @Override
            public void onCheckpointBegin(Context ctx) {
            /* No-op. */
            }
        };
        ((GridCacheDatabaseSharedManager) gridCtx.cache().context().database()).addCheckpointListener(checkpointLsnr);
    }
    lastPageIdx = 0;
    started = true;
    log.info("PageMemory tracker started, " + U.readableSize(maxMemorySize, false) + " offheap memory allocated.");
}