Examples with CheckpointEntry org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry used on opensource projects

Example 1 with CheckpointEntry

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method reserveHistoryForExchange.

/**
 * {@inheritDoc}
 */
@Override
public synchronized Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
    assert reservedForExchange == null : reservedForExchange;
    Map<Integer, Set<Integer>> /*partId*/
    applicableGroupsAndPartitions = partitionsApplicableForWalRebalance();
    Map<Integer, T2<ReservationReason, Map<Integer, CheckpointEntry>>> /*partId*/
    earliestValidCheckpoints;
    checkpointReadLock();
    WALPointer reservedCheckpointMark;
    try {
        CheckpointHistoryResult checkpointHistoryResult = checkpointHistory().searchAndReserveCheckpoints(applicableGroupsAndPartitions);
        earliestValidCheckpoints = checkpointHistoryResult.earliestValidCheckpoints();
        reservedForExchange = reservedCheckpointMark = checkpointHistoryResult.reservedCheckpointMark();
    } finally {
        checkpointReadUnlock();
    }
    Map<Integer, Map<Integer, Long>> /*updCntr*/
    grpPartsWithCnts = new HashMap<>();
    for (Map.Entry<Integer, T2<ReservationReason, Map<Integer, CheckpointEntry>>> /*partId*/
    e : earliestValidCheckpoints.entrySet()) {
        int grpId = e.getKey();
        if (e.getValue().get2() == null)
            continue;
        for (Map.Entry<Integer, CheckpointEntry> e0 : e.getValue().get2().entrySet()) {
            CheckpointEntry cpEntry = e0.getValue();
            int partId = e0.getKey();
            if (reservedCheckpointMark != null && !cctx.wal().reserved(reservedCheckpointMark)) {
                log.warning("Reservation failed because the segment was released: " + reservedCheckpointMark);
                reservedForExchange = null;
                grpPartsWithCnts.clear();
                return grpPartsWithCnts;
            }
            try {
                Long updCntr = cpEntry.partitionCounter(cctx.wal(), grpId, partId);
                if (updCntr != null)
                    grpPartsWithCnts.computeIfAbsent(grpId, k -> new HashMap<>()).put(partId, updCntr);
            } catch (IgniteCheckedException ex) {
                log.warning("Reservation failed because counters are not available [grpId=" + grpId + ", part=" + partId + ", cp=(" + cpEntry.checkpointId() + ", " + U.format(cpEntry.timestamp()) + ")]", ex);
            }
        }
    }
    if (log.isInfoEnabled() && !F.isEmpty(earliestValidCheckpoints))
        printReservationToLog(earliestValidCheckpoints);
    return grpPartsWithCnts;
}

Example 2 with CheckpointEntry

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry in project ignite by apache.

the class IgniteWalRecoveryTest method testBinaryRecoverBeforePMEWhenMiddleCheckpoint.

/**
 * Check binary recover completes successfully when node stopped at the middle of checkpoint. Destroy cache_data.bin
 * file for particular cache to emulate missing {@link DynamicCacheDescriptor} file (binary recovery should complete
 * successfully in this case).
 *
 * @throws Exception if failed.
 */
@Test
public void testBinaryRecoverBeforePMEWhenMiddleCheckpoint() throws Exception {
    startGrids(3);
    IgniteEx ig2 = grid(2);
    ig2.cluster().active(true);
    IgniteCache<Object, Object> cache = ig2.cache(CACHE_NAME);
    for (int i = 1; i <= 4_000; i++) cache.put(i, new BigObject(i));
    BigObject objToCheck;
    ig2.getOrCreateCache(CACHE_TO_DESTROY_NAME).put(1, objToCheck = new BigObject(1));
    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig2.context().cache().context().database();
    IgniteInternalFuture<?> cpFinishFut = dbMgr.forceCheckpoint("force checkpoint").futureFor(FINISHED);
    // Delete checkpoint END file to emulate node stopped at the middle of checkpoint.
    cpFinishFut.listen(new IgniteInClosureX<IgniteInternalFuture>() {

        @Override
        public void applyx(IgniteInternalFuture fut0) throws IgniteCheckedException {
            try {
                CheckpointEntry cpEntry = dbMgr.checkpointHistory().lastCheckpoint();
                String cpEndFileName = CheckpointMarkersStorage.checkpointFileName(cpEntry, CheckpointEntryType.END);
                Files.delete(Paths.get(dbMgr.checkpointDirectory().getAbsolutePath(), cpEndFileName));
                log.info("Checkpoint marker removed [cpEndFileName=" + cpEndFileName + ']');
            } catch (IOException e) {
                throw new IgniteCheckedException(e);
            }
        }
    });
    // Resolve cache directory. Emulating cache destroy in the middle of checkpoint.
    IgniteInternalCache<Object, Object> destoryCache = ig2.cachex(CACHE_TO_DESTROY_NAME);
    FilePageStoreManager pageStoreMgr = (FilePageStoreManager) destoryCache.context().shared().pageStore();
    File destroyCacheWorkDir = pageStoreMgr.cacheWorkDir(destoryCache.configuration());
    // Stop the whole cluster
    stopAllGrids();
    // Delete cache_data.bin file for this cache. Binary recovery should complete successfully after it.
    final File[] files = destroyCacheWorkDir.listFiles(new FilenameFilter() {

        @Override
        public boolean accept(final File dir, final String name) {
            return name.endsWith(CACHE_DATA_FILENAME);
        }
    });
    assertTrue(files.length > 0);
    for (final File file : files) assertTrue("Can't remove " + file.getAbsolutePath(), file.delete());
    startGrids(2);
    // Preprare Ignite instance configuration with additional Discovery checks.
    final String ig2Name = getTestIgniteInstanceName(2);
    final IgniteConfiguration onJoinCfg = optimize(getConfiguration(ig2Name));
    // Check restore beeing called before PME and joining node to cluster.
    ((IgniteDiscoverySpi) onJoinCfg.getDiscoverySpi()).setInternalListener(new DiscoverySpiTestListener() {

        @Override
        public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
            String nodeName = locNode.attribute(ATTR_IGNITE_INSTANCE_NAME);
            GridCacheSharedContext sharedCtx = ((IgniteEx) ignite(getTestIgniteInstanceIndex(nodeName))).context().cache().context();
            if (nodeName.equals(ig2Name)) {
                // Checkpoint history initialized on node start.
                assertFalse(((GridCacheDatabaseSharedManager) sharedCtx.database()).checkpointHistory().checkpoints().isEmpty());
            }
            super.beforeJoin(locNode, log);
        }
    });
    Ignite restoredIg2 = startGrid(ig2Name, onJoinCfg);
    awaitPartitionMapExchange();
    assertEquals(restoredIg2.cache(CACHE_TO_DESTROY_NAME).get(1), objToCheck);
}

Example 3 with CheckpointEntry

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method reserveHistoryForPreloading.

/**
 * {@inheritDoc}
 */
@Override
public boolean reserveHistoryForPreloading(Map<T2<Integer, Integer>, Long> reservationMap) {
    Map<GroupPartitionId, CheckpointEntry> entries = checkpointHistory().searchCheckpointEntry(reservationMap);
    if (F.isEmpty(entries))
        return false;
    WALPointer oldestWALPointerToReserve = null;
    for (CheckpointEntry cpE : entries.values()) {
        WALPointer ptr = cpE.checkpointMark();
        if (ptr == null)
            return false;
        if (oldestWALPointerToReserve == null || ptr.compareTo(oldestWALPointerToReserve) < 0)
            oldestWALPointerToReserve = ptr;
    }
    if (cctx.wal().reserve(oldestWALPointerToReserve)) {
        reservedForPreloading.set(oldestWALPointerToReserve);
        return true;
    } else
        return false;
}

Example 4 with CheckpointEntry

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method printReservationToLog.

/**
 * Prints detail information about caches which were not reserved
 * and reservation depth for the caches which have WAL history enough.
 *
 * @param earliestValidCheckpoints Map contains information about caches' reservation.
 */
private void printReservationToLog(Map<Integer, T2<ReservationReason, Map<Integer, CheckpointEntry>>> earliestValidCheckpoints) {
    try {
        Map<ReservationReason, List<Integer>> notReservedCachesToPrint = new HashMap<>();
        Map<ReservationReason, List<T2<Integer, CheckpointEntry>>> reservedCachesToPrint = new HashMap<>();
        for (Map.Entry<Integer, T2<ReservationReason, Map<Integer, CheckpointEntry>>> entry : earliestValidCheckpoints.entrySet()) {
            if (entry.getValue().get2() == null) {
                notReservedCachesToPrint.computeIfAbsent(entry.getValue().get1(), reason -> new ArrayList<>()).add(entry.getKey());
            } else {
                reservedCachesToPrint.computeIfAbsent(entry.getValue().get1(), reason -> new ArrayList<>()).add(new T2(entry.getKey(), entry.getValue().get2().values().stream().min(Comparator.comparingLong(CheckpointEntry::timestamp)).get()));
            }
        }
        if (!F.isEmpty(notReservedCachesToPrint)) {
            log.info("Cache groups were not reserved [" + notReservedCachesToPrint.entrySet().stream().map(entry -> '[' + entry.getValue().stream().map(grpId -> "[grpId=" + grpId + ", grpName=" + cctx.cache().cacheGroup(grpId).cacheOrGroupName() + ']').collect(Collectors.joining(", ")) + ", reason=" + entry.getKey() + ']').collect(Collectors.joining(", ")) + ']');
        }
        if (!F.isEmpty(reservedCachesToPrint)) {
            log.info("Cache groups with earliest reserved checkpoint and a reason why a previous checkpoint was inapplicable: [" + reservedCachesToPrint.entrySet().stream().map(entry -> '[' + entry.getValue().stream().map(grpCp -> "[grpId=" + grpCp.get1() + ", grpName=" + cctx.cache().cacheGroup(grpCp.get1()).cacheOrGroupName() + ", cp=(" + grpCp.get2().checkpointId() + ", " + U.format(grpCp.get2().timestamp()) + ")]").collect(Collectors.joining(", ")) + ", reason=" + entry.getKey() + ']').collect(Collectors.joining(", ")) + ']');
        }
    } catch (Exception e) {
        log.error("An error happened during printing partitions that were reserved for potential historical rebalance.", e);
    }
}

Example 5 with CheckpointEntry

use of org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method lastCheckpointInapplicableForWalRebalance.

/**
 * Set last checkpoint as inapplicable for WAL rebalance for given group {@code grpId}.
 *
 * @param grpId Group ID.
 */
@Override
public void lastCheckpointInapplicableForWalRebalance(int grpId) {
    checkpointReadLock();
    try {
        CheckpointEntry lastCp = checkpointManager.checkpointMarkerStorage().removeFromEarliestCheckpoints(grpId);
        long lastCpTs = lastCp != null ? lastCp.timestamp() : 0;
        if (lastCpTs != 0)
            metaStorage.write(checkpointInapplicableCpAndGroupIdToKey(lastCpTs, grpId), true);
    } catch (IgniteCheckedException e) {
        log.error("Failed to mark last checkpoint as inapplicable for WAL rebalance for group: " + grpId, e);
    } finally {
        checkpointReadUnlock();
    }
}