Examples with CheckpointListener org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener used on opensource projects

Example 11 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class IgniteSnapshotManagerSelfTest method testSnapshotLocalPartitionMultiCpWithLoad.

/**
 * Test that all partitions are copied successfully even after multiple checkpoints occur during
 * the long copy of cache partition files.
 *
 * Data consistency checked through a test node started right from snapshot directory and all values
 * read successes.
 *
 * @throws Exception If fails.
 */
@Test
public void testSnapshotLocalPartitionMultiCpWithLoad() throws Exception {
    int valMultiplier = 2;
    CountDownLatch slowCopy = new CountDownLatch(1);
    // Start grid node with data before each test.
    IgniteEx ig = startGridsWithCache(1, CACHE_KEYS_RANGE, key -> new Account(key, key), new CacheConfiguration<>(DEFAULT_CACHE_NAME));
    GridCacheSharedContext<?, ?> cctx = ig.context().cache().context();
    AtomicInteger cntr = new AtomicInteger();
    CountDownLatch ldrLatch = new CountDownLatch(1);
    IgniteSnapshotManager mgr = snp(ig);
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) cctx.database();
    IgniteInternalFuture<?> loadFut = GridTestUtils.runMultiThreadedAsync(() -> {
        try {
            U.await(ldrLatch);
            while (!Thread.currentThread().isInterrupted()) ig.cache(DEFAULT_CACHE_NAME).put(cntr.incrementAndGet(), new Account(cntr.incrementAndGet(), cntr.incrementAndGet()));
        } catch (IgniteInterruptedCheckedException e) {
            log.warning("Loader has been interrupted", e);
        }
    }, 5, "cache-loader-");
    // Register task but not schedule it on the checkpoint.
    SnapshotFutureTask snpFutTask = (SnapshotFutureTask) mgr.registerSnapshotTask(SNAPSHOT_NAME, cctx.localNodeId(), F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) {

        @Override
        public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
            try {
                U.await(slowCopy);
                delegate.sendPart0(part, cacheDirName, pair, length);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteException(e);
            }
        }
    });
    db.addCheckpointListener(new CheckpointListener() {

        /**
         * {@inheritDoc}
         */
        @Override
        public void beforeCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onMarkCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onCheckpointBegin(Context ctx) {
            Map<Integer, Set<Integer>> processed = GridTestUtils.getFieldValue(snpFutTask, SnapshotFutureTask.class, "processed");
            if (!processed.isEmpty())
                ldrLatch.countDown();
        }
    });
    try {
        snpFutTask.start();
        // Change data before snapshot creation which must be included into it with correct value multiplier.
        for (int i = 0; i < CACHE_KEYS_RANGE; i++) ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, valMultiplier * i));
        // Snapshot is still in the INIT state. beforeCheckpoint has been skipped
        // due to checkpoint already running and we need to schedule the next one
        // right after current will be completed.
        cctx.database().forceCheckpoint(String.format(CP_SNAPSHOT_REASON, SNAPSHOT_NAME));
        snpFutTask.started().get();
        db.forceCheckpoint("snapshot is ready to be created").futureFor(CheckpointState.MARKER_STORED_TO_DISK).get();
        // Change data after snapshot.
        for (int i = 0; i < CACHE_KEYS_RANGE; i++) ig.cache(DEFAULT_CACHE_NAME).put(i, new Account(i, 3 * i));
        // Snapshot on the next checkpoint must copy page to delta file before write it to a partition.
        forceCheckpoint(ig);
        slowCopy.countDown();
        snpFutTask.get();
    } finally {
        loadFut.cancel();
    }
    // Now can stop the node and check created snapshots.
    stopGrid(0);
    cleanPersistenceDir(ig.name());
    // Start Ignite instance from snapshot directory.
    IgniteEx ig2 = startGridsFromSnapshot(1, SNAPSHOT_NAME);
    for (int i = 0; i < CACHE_KEYS_RANGE; i++) {
        assertEquals("snapshot data consistency violation [key=" + i + ']', i * valMultiplier, ((Account) ig2.cache(DEFAULT_CACHE_NAME).get(i)).balance);
    }
}

Example 12 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class IgniteSnapshotWithMetastorageTest method testMetastorageUpdateOnSnapshotFail.

/**
 * @throws Exception If fails.
 */
@Test
public void testMetastorageUpdateOnSnapshotFail() throws Exception {
    AtomicInteger keyCnt = new AtomicInteger();
    AtomicBoolean stop = new AtomicBoolean();
    Set<String> writtenKeys = new ConcurrentSkipListSet<>();
    IgniteEx ignite = startGridsWithCache(2, dfltCacheCfg, CACHE_KEYS_RANGE);
    IgniteInternalFuture<?> updFut = GridTestUtils.runMultiThreadedAsync(() -> {
        while (!Thread.currentThread().isInterrupted() && !stop.get()) {
            try {
                String key = SNAPSHOT_PREFIX + keyCnt.getAndIncrement();
                ignite.context().distributedMetastorage().write(key, "value");
                writtenKeys.add(key);
            } catch (IgniteCheckedException e) {
                throw new IgniteException(e);
            }
        }
    }, 3, "dms-updater");
    ((GridCacheDatabaseSharedManager) ignite.context().cache().context().database()).addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) {
            if (ctx.progress().reason().contains(SNAPSHOT_NAME)) {
                Map<String, SnapshotFutureTask> locMap = GridTestUtils.getFieldValue(snp(ignite), IgniteSnapshotManager.class, "locSnpTasks");
                // Fail the snapshot task with an exception, all metastorage keys must be successfully continued.
                locMap.get(SNAPSHOT_NAME).acceptException(new IgniteCheckedException("Test exception"));
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) {
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) {
        }
    });
    IgniteFuture<?> fut = ignite.snapshot().createSnapshot(SNAPSHOT_NAME);
    GridTestUtils.assertThrowsAnyCause(log, fut::get, IgniteCheckedException.class, "Test exception");
    stop.set(true);
    // Load future must complete without exceptions, all metastorage keys must be written.
    updFut.get();
    Set<String> readedKeys = new TreeSet<>();
    ignite.context().distributedMetastorage().iterate(SNAPSHOT_PREFIX, (key, val) -> readedKeys.add(key));
    assertEquals("Not all metastorage keys have been written", writtenKeys, readedKeys);
}

Example 13 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class DurableBackgroundTasksProcessorSelfTest method testDontDeleteTaskIfItsRestart.

/**
 * Check that the task will not be deleted from the MetaStorage if it was restarted.
 *
 * @throws Exception If failed.
 */
@Test
public void testDontDeleteTaskIfItsRestart() throws Exception {
    IgniteEx n = startGrid(0);
    ObservingCheckpointListener observingCpLsnr = new ObservingCheckpointListener();
    dbMgr(n).addCheckpointListener(observingCpLsnr);
    n.cluster().state(ACTIVE);
    CheckpointWorkflow cpWorkflow = checkpointWorkflow(n);
    List<CheckpointListener> cpLs = cpWorkflow.getRelevantCheckpointListeners(dbMgr(n).checkpointedDataRegions());
    assertTrue(cpLs.contains(observingCpLsnr));
    assertTrue(cpLs.contains(durableBackgroundTask(n)));
    assertTrue(cpLs.indexOf(observingCpLsnr) < cpLs.indexOf(durableBackgroundTask(n)));
    SimpleTask simpleTask0 = new SimpleTask("t");
    IgniteInternalFuture<Void> taskFut = durableBackgroundTask(n).executeAsync(simpleTask0, true);
    simpleTask0.onExecFut.get(getTestTimeout());
    forceCheckpoint();
    dbMgr(n).enableCheckpoints(false).get(getTestTimeout());
    simpleTask0.taskFut.onDone(DurableBackgroundTaskResult.complete(null));
    taskFut.get(getTestTimeout());
    SimpleTask simpleTask1 = new SimpleTask("t");
    AtomicReference<IgniteInternalFuture<Void>> taskFutRef = new AtomicReference<>();
    observingCpLsnr.afterCheckpointEndConsumer = ctx -> taskFutRef.set(durableBackgroundTask(n).executeAsync(simpleTask1, true));
    dbMgr(n).enableCheckpoints(true).get(getTestTimeout());
    forceCheckpoint();
    assertNotNull(metaStorageOperation(n, ms -> ms.read(metaStorageKey(simpleTask0))));
    simpleTask1.onExecFut.get(getTestTimeout());
    simpleTask1.taskFut.onDone(DurableBackgroundTaskResult.complete(null));
    taskFutRef.get().get(getTestTimeout());
    forceCheckpoint();
    assertNull(metaStorageOperation(n, ms -> ms.read(metaStorageKey(simpleTask1))));
}

Example 14 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class DurableBackgroundTasksProcessorSelfTest method testNotFailNodeWhenNodeStoppindAndDeleteTasks.

/**
 * Checks that stopping the node does not fail the node when deleting tasks.
 *
 * @throws Exception If failed.
 */
@Test
public void testNotFailNodeWhenNodeStoppindAndDeleteTasks() throws Exception {
    IgniteEx n = startGrid(0);
    ObservingCheckpointListener observingCpLsnr = new ObservingCheckpointListener();
    dbMgr(n).addCheckpointListener(observingCpLsnr);
    n.cluster().state(ACTIVE);
    CheckpointWorkflow cpWorkflow = checkpointWorkflow(n);
    List<CheckpointListener> cpLs = cpWorkflow.getRelevantCheckpointListeners(dbMgr(n).checkpointedDataRegions());
    assertTrue(cpLs.contains(observingCpLsnr));
    assertTrue(cpLs.contains(durableBackgroundTask(n)));
    assertTrue(cpLs.indexOf(observingCpLsnr) < cpLs.indexOf(durableBackgroundTask(n)));
    SimpleTask simpleTask = new SimpleTask("t");
    IgniteInternalFuture<Void> taskFut = durableBackgroundTask(n).executeAsync(simpleTask, true);
    simpleTask.onExecFut.get(getTestTimeout());
    GridFutureAdapter<Void> startStopFut = new GridFutureAdapter<>();
    GridFutureAdapter<Void> finishStopFut = new GridFutureAdapter<>();
    observingCpLsnr.repeatOnMarkCheckpointBeginConsumer = true;
    observingCpLsnr.onMarkCheckpointBeginConsumer = ctx -> {
        if (n.context().isStopping()) {
            startStopFut.onDone();
            finishStopFut.get(getTestTimeout());
            observingCpLsnr.repeatOnMarkCheckpointBeginConsumer = false;
        }
    };
    IgniteInternalFuture<Void> stopFut = runAsync(() -> stopAllGrids(false));
    startStopFut.get(getTestTimeout());
    simpleTask.taskFut.onDone(DurableBackgroundTaskResult.complete(null));
    taskFut.get(getTestTimeout());
    finishStopFut.onDone();
    stopFut.get(getTestTimeout());
    assertNull(n.context().failure().failureContext());
    assertThrows(log, () -> durableBackgroundTask(n).executeAsync(simpleTask, true), IgniteException.class, null);
}

Example 15 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class PageMemoryTracker method start.

/**
 * Start tracking pages.
 */
synchronized void start() {
    if (!isEnabled() || started)
        return;
    pageSize = ctx.igniteConfiguration().getDataStorageConfiguration().getPageSize();
    pageMemoryMock = mockPageMemory();
    GridCacheSharedContext sharedCtx = gridCtx.cache().context();
    // Initialize one memory region for all data regions of target ignite node.
    long maxMemorySize = 0;
    for (DataRegion dataRegion : sharedCtx.database().dataRegions()) {
        if (dataRegion.pageMemory() instanceof PageMemoryImpl)
            maxMemorySize += dataRegion.config().getMaxSize();
    }
    long[] chunks = new long[] { maxMemorySize };
    memoryProvider = new UnsafeMemoryProvider(log);
    memoryProvider.initialize(chunks);
    memoryRegion = memoryProvider.nextRegion();
    GridUnsafe.setMemory(memoryRegion.address(), memoryRegion.size(), (byte) 0);
    maxPages = (int) (maxMemorySize / pageSize);
    pageSlots = new DirectMemoryPageSlot[maxPages];
    freeSlotsCnt = maxPages;
    tmpBuf1 = ByteBuffer.allocateDirect(pageSize);
    tmpBuf2 = ByteBuffer.allocateDirect(pageSize);
    if (cfg.isCheckPagesOnCheckpoint()) {
        checkpointLsnr = new CheckpointListener() {

            @Override
            public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
                if (!checkPages(false, true))
                    throw new IgniteCheckedException("Page memory is inconsistent after applying WAL delta records.");
            }

            @Override
            public void beforeCheckpointBegin(Context ctx) {
            /* No-op. */
            }

            @Override
            public void onCheckpointBegin(Context ctx) {
            /* No-op. */
            }
        };
        ((GridCacheDatabaseSharedManager) gridCtx.cache().context().database()).addCheckpointListener(checkpointLsnr);
    }
    lastPageIdx = 0;
    started = true;
    log.info("PageMemory tracker started, " + U.readableSize(maxMemorySize, false) + " offheap memory allocated.");
}