Examples with CheckpointListener org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener used on opensource projects

Example 6 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project gridgain by gridgain.

the class IgnitePdsConsistencyOnDelayedPartitionOwning method checkConsistencyNodeLeft.

/**
 */
@Test
public void checkConsistencyNodeLeft() throws Exception {
    IgniteEx crd = (IgniteEx) startGridsMultiThreaded(4);
    crd.cluster().active(true);
    for (int i = 0; i < PARTS; i++) crd.cache(DEFAULT_CACHE_NAME).put(i, i);
    forceCheckpoint();
    stopGrid(1);
    for (int i = 0; i < PARTS; i++) crd.cache(DEFAULT_CACHE_NAME).put(i, i + 1);
    // Block supply messages from all owners.
    TestRecordingCommunicationSpi spi0 = TestRecordingCommunicationSpi.spi(grid(0));
    TestRecordingCommunicationSpi spi2 = TestRecordingCommunicationSpi.spi(grid(2));
    TestRecordingCommunicationSpi spi3 = TestRecordingCommunicationSpi.spi(grid(3));
    IgniteBiPredicate<ClusterNode, Message> pred = new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode clusterNode, Message msg) {
            return msg instanceof GridDhtPartitionSupplyMessage;
        }
    };
    spi0.blockMessages(pred);
    spi2.blockMessages(pred);
    spi3.blockMessages(pred);
    GridTestUtils.runAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            startGrid(1);
            return null;
        }
    });
    spi0.waitForBlocked();
    spi2.waitForBlocked();
    spi3.waitForBlocked();
    spi0.stopBlock();
    spi2.stopBlock();
    CountDownLatch topInitLatch = new CountDownLatch(1);
    CountDownLatch enableDurabilityCPStartLatch = new CountDownLatch(1);
    CountDownLatch delayedOnwningLatch = new CountDownLatch(1);
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) grid(1).context().cache().context().database();
    dbMgr.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
        // No-op.
        }

        @Override
        public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
        // No-op.
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
            String reason = ctx.progress().reason();
            String reason0 = WalStateManager.reason(cacheId(DEFAULT_CACHE_NAME), new AffinityTopologyVersion(6, 0));
            if (reason != null && reason.equals(reason0)) {
                enableDurabilityCPStartLatch.countDown();
                try {
                    assertTrue(U.await(delayedOnwningLatch, 10_000, TimeUnit.MILLISECONDS));
                } catch (IgniteInterruptedCheckedException e) {
                    fail(X.getFullStackTrace(e));
                }
            }
        }
    });
    TestRecordingCommunicationSpi.spi(grid(1)).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode clusterNode, Message msg) {
            if (msg instanceof GridDhtPartitionDemandMessage) {
                GridDhtPartitionDemandMessage msg0 = (GridDhtPartitionDemandMessage) msg;
                return msg0.topologyVersion().equals(new AffinityTopologyVersion(7, 0));
            }
            return false;
        }
    });
    grid(1).context().cache().context().exchange().registerExchangeAwareComponent(new PartitionsExchangeAware() {

        @Override
        public void onDoneBeforeTopologyUnlock(GridDhtPartitionsExchangeFuture fut) {
            if (fut.initialVersion().equals(new AffinityTopologyVersion(7, 0))) {
                topInitLatch.countDown();
                try {
                    assertTrue(U.await(enableDurabilityCPStartLatch, 20_000, TimeUnit.MILLISECONDS));
                } catch (IgniteInterruptedCheckedException e) {
                    fail(X.getFullStackTrace(e));
                }
                System.out.println();
            }
        }
    });
    // Trigger rebalancing remap because owner has left.
    IgniteInternalFuture stopFut = GridTestUtils.runAsync(new Runnable() {

        @Override
        public void run() {
            // TODO start cache.
            stopGrid(2);
        }
    });
    // Wait for topology (7,0) init on grid1 before finishing rebalancing on (6,0).
    assertTrue(U.await(topInitLatch, 20_000, TimeUnit.MILLISECONDS));
    // Release last supply message, causing triggering a cp for enabling durability.
    spi3.stopBlock();
    // Wait for new rebalancing assignments ready on grid1.
    TestRecordingCommunicationSpi.spi(grid(1)).waitForBlocked();
    // Triggers spurious ideal switching before rebalancing has finished for (7,0).
    delayedOnwningLatch.countDown();
    stopFut.get();
    TestRecordingCommunicationSpi.spi(grid(1)).stopBlock();
    awaitPartitionMapExchange();
    assertPartitionsSame(idleVerify(grid(0), DEFAULT_CACHE_NAME));
    for (GridDhtPartitionsExchangeFuture fut : grid(0).context().cache().context().exchange().exchangeFutures()) assertTrue(fut.toString(), fut.invalidPartitions().isEmpty());
    CacheGroupContext grpCtx = grid(1).context().cache().cacheGroup(cacheId(DEFAULT_CACHE_NAME));
    if (grpCtx != null)
        assertTrue(grpCtx.localWalEnabled());
}

Example 7 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project gridgain by gridgain.

the class CheckpointFreeListTest method testFreeListUnderLoadMultipleCheckpoints.

/**
 * Test checks that free-list works and pages cache flushes correctly under the high concurrent load.
 */
@Test
public void testFreeListUnderLoadMultipleCheckpoints() throws Throwable {
    IgniteEx ignite = startGrid(0);
    ignite.cluster().active(true);
    int minValSize = 64;
    int maxValSize = 128;
    int valsCnt = maxValSize - minValSize;
    int keysCnt = 1_000;
    byte[][] vals = new byte[valsCnt][];
    for (int i = 0; i < valsCnt; i++) vals[i] = new byte[minValSize + i];
    IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME).setAffinity(// Maximize contention per partition.
    new RendezvousAffinityFunction().setPartitions(2)).setAtomicityMode(CacheAtomicityMode.ATOMIC));
    AtomicBoolean done = new AtomicBoolean();
    AtomicReference<Throwable> error = new AtomicReference<>();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) ignite.context().cache().context().database();
    AtomicLong pageListCacheLimitHolder = db.pageListCacheLimitHolder(ignite.context().cache().cache(DEFAULT_CACHE_NAME).context().dataRegion());
    long initPageListCacheLimit = pageListCacheLimitHolder.get();
    // Add listener after cache is started, so this listener will be triggered after listener for cache.
    db.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
            // Need to wait for condition here, since checkpointer can store free-list metadata asynchronously.
            if (!waitForCondition(() -> initPageListCacheLimit == pageListCacheLimitHolder.get(), 1_000L)) {
                IgniteCheckedException e = new IgniteCheckedException("Page list cache limit doesn't restored " + "correctly [init=" + initPageListCacheLimit + ", cur=" + pageListCacheLimitHolder.get() + ']');
                error.set(e);
                throw e;
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) {
        // No-op.
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) {
        // No-op.
        }
    });
    IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(() -> {
        Random rnd = new Random();
        try {
            while (!done.get()) {
                int key = rnd.nextInt(keysCnt);
                byte[] val = vals[rnd.nextInt(valsCnt)];
                // Put with changed value size - worst case for free list, since row will be removed first and
                // then inserted again.
                cache.put(key, val);
            }
        } catch (Throwable t) {
            error.set(t);
        }
    }, 20, "cache-put");
    for (int i = 0; i < SF.applyLB(10, 2); i++) {
        if (error.get() != null)
            break;
        forceCheckpoint(ignite);
        doSleep(1_000L);
    }
    done.set(true);
    fut.get();
    stopAllGrids();
    if (error.get() != null)
        throw error.get();
}

Example 8 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project gridgain by gridgain.

the class ResumeCreateIndexTest method awaitBeforeCheckpointBeginAsync.

/**
 * Waiting for a {@link CheckpointListener#beforeCheckpointBegin} asynchronously
 * for a checkpoint for a specific reason.
 *
 * @param n Node.
 * @param reason Checkpoint reason.
 * @return Future for waiting for the {@link CheckpointListener#beforeCheckpointBegin}.
 */
private IgniteInternalFuture<Void> awaitBeforeCheckpointBeginAsync(IgniteEx n, String reason) {
    GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
    dbMgr(n).addCheckpointListener(new CheckpointListener() {

        /**
         * {@inheritDoc}
         */
        @Override
        public void onMarkCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void onCheckpointBegin(Context ctx) {
        // No-op.
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void beforeCheckpointBegin(Context ctx) {
            if (reason.equals(ctx.progress().reason()))
                fut.onDone();
        }
    });
    return fut;
}

Example 9 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class IgnitePdsConsistencyOnDelayedPartitionOwning method checkConsistencyNodeLeft.

/**
 */
@Test
public void checkConsistencyNodeLeft() throws Exception {
    IgniteEx crd = (IgniteEx) startGridsMultiThreaded(4);
    crd.cluster().active(true);
    for (int i = 0; i < PARTS; i++) crd.cache(DEFAULT_CACHE_NAME).put(i, i);
    forceCheckpoint();
    stopGrid(1);
    for (int i = 0; i < PARTS; i++) crd.cache(DEFAULT_CACHE_NAME).put(i, i + 1);
    // Block supply messages from all owners.
    TestRecordingCommunicationSpi spi0 = TestRecordingCommunicationSpi.spi(grid(0));
    TestRecordingCommunicationSpi spi2 = TestRecordingCommunicationSpi.spi(grid(2));
    TestRecordingCommunicationSpi spi3 = TestRecordingCommunicationSpi.spi(grid(3));
    IgniteBiPredicate<ClusterNode, Message> pred = new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode clusterNode, Message msg) {
            return msg instanceof GridDhtPartitionSupplyMessage;
        }
    };
    spi0.blockMessages(pred);
    spi2.blockMessages(pred);
    spi3.blockMessages(pred);
    GridTestUtils.runAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            startGrid(1);
            return null;
        }
    });
    spi0.waitForBlocked();
    spi2.waitForBlocked();
    spi3.waitForBlocked();
    spi0.stopBlock();
    spi2.stopBlock();
    CountDownLatch topInitLatch = new CountDownLatch(1);
    CountDownLatch enableDurabilityCPStartLatch = new CountDownLatch(1);
    CountDownLatch delayedOnwningLatch = new CountDownLatch(1);
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) grid(1).context().cache().context().database();
    dbMgr.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
        // No-op.
        }

        @Override
        public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
        // No-op.
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
            String reason = ctx.progress().reason();
            String reason0 = WalStateManager.reason(cacheId(DEFAULT_CACHE_NAME), new AffinityTopologyVersion(6, 0));
            if (reason != null && reason.equals(reason0)) {
                enableDurabilityCPStartLatch.countDown();
                try {
                    assertTrue(U.await(delayedOnwningLatch, 10_000, TimeUnit.MILLISECONDS));
                } catch (IgniteInterruptedCheckedException e) {
                    fail(X.getFullStackTrace(e));
                }
            }
        }
    });
    TestRecordingCommunicationSpi.spi(grid(1)).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {

        @Override
        public boolean apply(ClusterNode clusterNode, Message msg) {
            if (msg instanceof GridDhtPartitionDemandMessage) {
                GridDhtPartitionDemandMessage msg0 = (GridDhtPartitionDemandMessage) msg;
                return msg0.topologyVersion().equals(new AffinityTopologyVersion(7, 0));
            }
            return false;
        }
    });
    grid(1).context().cache().context().exchange().registerExchangeAwareComponent(new PartitionsExchangeAware() {

        @Override
        public void onDoneBeforeTopologyUnlock(GridDhtPartitionsExchangeFuture fut) {
            if (fut.initialVersion().equals(new AffinityTopologyVersion(7, 0))) {
                topInitLatch.countDown();
                try {
                    assertTrue(U.await(enableDurabilityCPStartLatch, 20_000, TimeUnit.MILLISECONDS));
                } catch (IgniteInterruptedCheckedException e) {
                    fail(X.getFullStackTrace(e));
                }
                System.out.println();
            }
        }
    });
    // Trigger rebalancing remap because owner has left.
    IgniteInternalFuture stopFut = GridTestUtils.runAsync(new Runnable() {

        @Override
        public void run() {
            // TODO start cache.
            stopGrid(2);
        }
    });
    // Wait for topology (7,0) init on grid1 before finishing rebalancing on (6,0).
    assertTrue(U.await(topInitLatch, 20_000, TimeUnit.MILLISECONDS));
    // Release last supply message, causing triggering a cp for enabling durability.
    spi3.stopBlock();
    // Wait for new rebalancing assignments ready on grid1.
    TestRecordingCommunicationSpi.spi(grid(1)).waitForBlocked();
    // Triggers spurious ideal switching before rebalancing has finished for (7,0).
    delayedOnwningLatch.countDown();
    stopFut.get();
    TestRecordingCommunicationSpi.spi(grid(1)).stopBlock();
    awaitPartitionMapExchange();
    assertPartitionsSame(idleVerify(grid(0), DEFAULT_CACHE_NAME));
    CacheGroupContext grpCtx = grid(1).context().cache().cacheGroup(cacheId(DEFAULT_CACHE_NAME));
    if (grpCtx != null)
        assertTrue(grpCtx.localWalEnabled());
}

Example 10 with CheckpointListener

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointListener in project ignite by apache.

the class IgniteClusterSnapshotCheckTest method testClusterSnapshotCheckFailsOnPartitionDataDiffers.

/**
 * @throws Exception If fails.
 */
@Test
public void testClusterSnapshotCheckFailsOnPartitionDataDiffers() throws Exception {
    CacheConfiguration<Integer, Value> ccfg = txCacheConfig(new CacheConfiguration<Integer, Value>(DEFAULT_CACHE_NAME)).setAffinity(new RendezvousAffinityFunction(false, 1));
    IgniteEx ignite = startGridsWithoutCache(2);
    ignite.getOrCreateCache(ccfg).put(1, new Value(new byte[2000]));
    forceCheckpoint(ignite);
    GridCacheSharedContext<?, ?> cctx = ignite.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) cctx.database();
    BinaryContext binCtx = ((CacheObjectBinaryProcessorImpl) ignite.context().cacheObjects()).binaryContext();
    GridCacheAdapter<?, ?> cache = ignite.context().cache().internalCache(dfltCacheCfg.getName());
    long partCtr = cache.context().topology().localPartition(PART_ID, NONE, false).dataStore().updateCounter();
    AtomicBoolean done = new AtomicBoolean();
    db.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
            // Change the cache value only at on of the cluster node to get hash conflict when the check command ends.
            if (!done.compareAndSet(false, true))
                return;
            GridIterator<CacheDataRow> it = cache.context().offheap().partitionIterator(PART_ID);
            assertTrue(it.hasNext());
            CacheDataRow row0 = it.nextX();
            AffinityTopologyVersion topVer = cctx.exchange().readyAffinityVersion();
            GridCacheEntryEx cached = cache.entryEx(row0.key(), topVer);
            byte[] bytes = new byte[2000];
            new Random().nextBytes(bytes);
            try {
                BinaryObjectImpl newVal = new BinaryObjectImpl(binCtx, binCtx.marshaller().marshal(new Value(bytes)), 0);
                boolean success = cached.initialValue(newVal, new GridCacheVersion(row0.version().topologyVersion(), row0.version().nodeOrder(), row0.version().order() + 1), null, null, TxState.NA, TxState.NA, TTL_ETERNAL, row0.expireTime(), true, topVer, DR_NONE, false, false, null);
                assertTrue(success);
                long newPartCtr = cache.context().topology().localPartition(PART_ID, NONE, false).dataStore().updateCounter();
                assertEquals(newPartCtr, partCtr);
            } catch (Exception e) {
                throw new IgniteCheckedException(e);
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
        }
    });
    db.waitForCheckpoint("test-checkpoint");
    ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get();
    Path part0 = U.searchFileRecursively(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME).toPath(), getPartitionFileName(PART_ID));
    assertNotNull(part0);
    assertTrue(part0.toString(), part0.toFile().exists());
    IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get();
    StringBuilder b = new StringBuilder();
    res.print(b::append, true);
    assertTrue(F.isEmpty(res.exceptions()));
    assertContains(log, b.toString(), "The check procedure has failed, conflict partitions has been found: [counterConflicts=0, hashConflicts=1]");
}