Examples with CheckpointProgress org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress used on opensource projects

Example 6 with CheckpointProgress

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress in project ignite by apache.

the class BPlusTreeReuseListPageMemoryImplTest method createPageMemory.

/**
 * {@inheritDoc}
 */
@Override
protected PageMemory createPageMemory() throws Exception {
    long[] sizes = new long[CPUS + 1];
    for (int i = 0; i < sizes.length; i++) sizes[i] = 1024 * MB / CPUS;
    sizes[CPUS] = 10 * MB;
    DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
    IgniteConfiguration cfg = new IgniteConfiguration();
    cfg.setEncryptionSpi(new NoopEncryptionSpi());
    cfg.setMetricExporterSpi(new NoopMetricExporterSpi());
    cfg.setSystemViewExporterSpi(new JmxSystemViewExporterSpi());
    cfg.setDataStorageConfiguration(new DataStorageConfiguration());
    GridTestKernalContext cctx = new GridTestKernalContext(log, cfg);
    cctx.add(new IgnitePluginProcessor(cctx, cfg, Collections.emptyList()));
    cctx.add(new GridInternalSubscriptionProcessor(cctx));
    cctx.add(new PerformanceStatisticsProcessor(cctx));
    cctx.add(new GridEncryptionManager(cctx));
    cctx.add(new GridMetricManager(cctx));
    cctx.add(new GridSystemViewManager(cctx));
    GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(cctx, null, null, null, new NoOpPageStoreManager(), new NoOpWALManager(), null, new IgniteCacheDatabaseSharedManager(), null, null, null, null, null, null, null, null, null, null, null, null, null);
    IgniteOutClosure<CheckpointProgress> clo = new IgniteOutClosure<CheckpointProgress>() {

        @Override
        public CheckpointProgress apply() {
            return Mockito.mock(CheckpointProgressImpl.class);
        }
    };
    PageMemory mem = new PageMemoryImpl(provider, sizes, sharedCtx, sharedCtx.pageStore(), PAGE_SIZE, (fullPageId, byteBuf, tag) -> {
        assert false : "No page replacement (rotation with disk) should happen during the test";
    }, new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {

        @Override
        public void applyx(Long page, FullPageId fullPageId, PageMemoryEx pageMem) {
        }
    }, () -> true, new DataRegionMetricsImpl(new DataRegionConfiguration(), cctx), PageMemoryImpl.ThrottlingPolicy.DISABLED, clo);
    mem.start();
    return mem;
}

Example 7 with CheckpointProgress

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress in project ignite by apache.

the class IgniteThrottlingUnitTest method wakeupSpeedBaseThrottledThreadOnCheckpointFinish.

/**
 * @throws IgniteInterruptedCheckedException if failed.
 */
@Test
public void wakeupSpeedBaseThrottledThreadOnCheckpointFinish() throws IgniteInterruptedCheckedException {
    // given: Enabled throttling with EXPONENTIAL level.
    CheckpointProgressImpl cl0 = mock(CheckpointProgressImpl.class);
    when(cl0.writtenPagesCounter()).thenReturn(new AtomicInteger(200));
    IgniteOutClosure<CheckpointProgress> cpProgress = mock(IgniteOutClosure.class);
    when(cpProgress.apply()).thenReturn(cl0);
    PagesWriteThrottlePolicy plc = new PagesWriteSpeedBasedThrottle(pageMemory2g, cpProgress, stateChecker, log) {

        @Override
        protected void doPark(long throttleParkTimeNs) {
            // Force parking to long time.
            super.doPark(TimeUnit.SECONDS.toNanos(1));
        }
    };
    simulateCheckpointBufferInDangerZoneSituation();
    AtomicBoolean stopLoad = new AtomicBoolean();
    List<Thread> loadThreads = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        loadThreads.add(new Thread(() -> {
            while (!stopLoad.get()) plc.onMarkDirty(true);
        }, "load-" + i));
    }
    try {
        loadThreads.forEach(Thread::start);
        // and: All load threads are parked.
        for (Thread t : loadThreads) assertTrue(t.getName(), waitForCondition(() -> t.getState() == TIMED_WAITING, 1000L));
        // when: Disable throttling
        simulateCheckpointBufferInSafeZoneSituation();
        stopReportingCheckpointProgress(cpProgress);
        // and: Finish the checkpoint.
        plc.onFinishCheckpoint();
        // then: All load threads should be unparked.
        for (Thread t : loadThreads) assertTrue(t.getName(), waitForCondition(() -> t.getState() != TIMED_WAITING, 500L));
        for (Thread t : loadThreads) assertNotEquals(t.getName(), TIMED_WAITING, t.getState());
    } finally {
        stopLoad.set(true);
    }
}

Example 8 with CheckpointProgress

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress in project ignite by apache.

the class WalStateManager method onProposeExchange.

/**
 * Handle propose message which is synchronized with other cache state actions through exchange thread.
 * If operation is no-op (i.e. state is not changed), then no additional processing is needed, and coordinator will
 * trigger finish request right away. Otherwise all nodes start asynchronous checkpoint flush, and send responses
 * to coordinator. Once all responses are received, coordinator node will trigger finish message.
 *
 * @param msg Message.
 */
public void onProposeExchange(WalStateProposeMessage msg) {
    if (!srv)
        return;
    synchronized (mux) {
        WalStateResult res = null;
        if (msg.affinityNode()) {
            // Affinity node, normal processing.
            CacheGroupContext grpCtx = cacheProcessor().cacheGroup(msg.groupId());
            if (grpCtx == null) {
                // Related caches were destroyed concurrently.
                res = new WalStateResult(msg, "Failed to change WAL mode because some caches " + "no longer exist: " + msg.caches().keySet());
            } else {
                if (F.eq(msg.enable(), grpCtx.globalWalEnabled()))
                    // Nothing changed -> no-op.
                    res = new WalStateResult(msg, false);
                else {
                    // Initiate a checkpoint.
                    CheckpointProgress cpFut = triggerCheckpoint("wal-state-change-grp-" + msg.groupId());
                    if (cpFut != null) {
                        try {
                            // Wait for checkpoint mark synchronously before releasing the control.
                            cpFut.futureFor(LOCK_RELEASED).get();
                            if (msg.enable()) {
                                grpCtx.globalWalEnabled(true);
                                // Enable: it is enough to release cache operations once mark is finished because
                                // not-yet-flushed dirty pages have been logged.
                                WalStateChangeWorker worker = new WalStateChangeWorker(msg, cpFut);
                                IgniteThread thread = new IgniteThread(worker);
                                thread.setUncaughtExceptionHandler(new OomExceptionHandler(cctx.kernalContext()));
                                thread.start();
                            } else {
                                // Disable: not-yet-flushed operations are not logged, so wait for them
                                // synchronously in exchange thread. Otherwise, we cannot define a point in
                                // when it is safe to continue cache operations.
                                res = awaitCheckpoint(cpFut, msg);
                                // WAL state is persisted after checkpoint if finished. Otherwise in case of crash
                                // and restart we will think that WAL is enabled, but data might be corrupted.
                                grpCtx.globalWalEnabled(false);
                            }
                        } catch (Exception e) {
                            U.warn(log, "Failed to change WAL mode due to unexpected exception [" + "msg=" + msg + ']', e);
                            res = new WalStateResult(msg, "Failed to change WAL mode due to unexpected " + "exception (see server logs for more information): " + e.getMessage());
                        }
                    } else {
                        res = new WalStateResult(msg, "Failed to initiate a checkpoint (checkpoint thread " + "is not available).");
                    }
                }
            }
        } else {
            // We cannot know result on non-affinity server node, so just complete operation with "false" flag,
            // which will be ignored anyway.
            res = new WalStateResult(msg, false);
        }
        if (res != null) {
            addResult(res);
            onCompletedLocally(res);
        }
    }
}

Example 9 with CheckpointProgress

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress in project ignite by apache.

the class PagesWriteThrottle method onMarkDirty.

/**
 * {@inheritDoc}
 */
@Override
public void onMarkDirty(boolean isPageInCheckpoint) {
    assert stateChecker.checkpointLockIsHeldByThread();
    boolean shouldThrottle = false;
    if (isPageInCheckpoint)
        shouldThrottle = isCpBufferOverflowThresholdExceeded();
    if (!shouldThrottle && !throttleOnlyPagesInCheckpoint) {
        CheckpointProgress progress = cpProgress.apply();
        AtomicInteger writtenPagesCntr = progress == null ? null : cpProgress.apply().writtenPagesCounter();
        if (progress == null || writtenPagesCntr == null)
            // Don't throttle if checkpoint is not running.
            return;
        int cpWrittenPages = writtenPagesCntr.get();
        int cpTotalPages = progress.currentCheckpointPagesCount();
        if (cpWrittenPages == cpTotalPages) {
            // Checkpoint is already in fsync stage, increasing maximum ratio of dirty pages to 3/4
            shouldThrottle = pageMemory.shouldThrottle(3.0 / 4);
        } else {
            double dirtyRatioThreshold = ((double) cpWrittenPages) / cpTotalPages;
            // Starting with 0.05 to avoid throttle right after checkpoint start
            // 7/12 is maximum ratio of dirty pages
            dirtyRatioThreshold = (dirtyRatioThreshold * 0.95 + 0.05) * 7 / 12;
            shouldThrottle = pageMemory.shouldThrottle(dirtyRatioThreshold);
        }
    }
    ExponentialBackoffThrottlingStrategy exponentialThrottle = isPageInCheckpoint ? inCheckpointProtection : notInCheckpointProtection;
    if (shouldThrottle) {
        long throttleParkTimeNs = exponentialThrottle.protectionParkTime();
        Thread curThread = Thread.currentThread();
        if (throttleParkTimeNs > LOGGING_THRESHOLD) {
            U.warn(log, "Parking thread=" + curThread.getName() + " for timeout(ms)=" + (throttleParkTimeNs / 1_000_000));
        }
        long startTime = U.currentTimeMillis();
        if (isPageInCheckpoint) {
            cpBufThrottledThreads.put(curThread.getId(), curThread);
            try {
                LockSupport.parkNanos(throttleParkTimeNs);
            } finally {
                cpBufThrottledThreads.remove(curThread.getId());
                if (throttleParkTimeNs > LOGGING_THRESHOLD) {
                    U.warn(log, "Unparking thread=" + curThread.getName() + " with park timeout(ms)=" + (throttleParkTimeNs / 1_000_000));
                }
            }
        } else
            LockSupport.parkNanos(throttleParkTimeNs);
        pageMemory.metrics().addThrottlingTime(U.currentTimeMillis() - startTime);
    } else {
        boolean backoffWasAlreadyStarted = exponentialThrottle.resetBackoff();
        if (isPageInCheckpoint && backoffWasAlreadyStarted)
            unparkParkedThreads();
    }
}

Example 10 with CheckpointProgress

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointProgress in project ignite by apache.

the class GridCacheDatabaseSharedManager method onStateRestored.

/**
 * Called when all partitions have been fully restored and pre-created on node start.
 *
 * Starts checkpointing process and initiates first checkpoint.
 *
 * @throws IgniteCheckedException If first checkpoint has failed.
 */
@Override
public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException {
    checkpointManager.start();
    CheckpointProgress chp = checkpointManager.forceCheckpoint("node started", null);
    if (chp != null)
        chp.futureFor(LOCK_RELEASED).get();
}