Examples with FilePageStoreManager.cacheGroupName org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName used on opensource projects

Example 1 with FilePageStoreManager.cacheGroupName

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName in project ignite by apache.

the class SnapshotRestoreProcess method preload.

/**
 * @param reqId Request id.
 * @return Future which will be completed when the preload ends.
 */
private IgniteInternalFuture<Boolean> preload(UUID reqId) {
    if (ctx.clientNode())
        return new GridFinishedFuture<>();
    SnapshotRestoreContext opCtx0 = opCtx;
    GridFutureAdapter<Boolean> retFut = new GridFutureAdapter<>();
    if (opCtx0 == null)
        return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot restore process has incorrect restore state: " + reqId));
    if (opCtx0.dirs.isEmpty())
        return new GridFinishedFuture<>();
    try {
        if (ctx.isStopping())
            throw new NodeStoppingException("Node is stopping: " + ctx.localNodeId());
        Set<SnapshotMetadata> allMetas = opCtx0.metasPerNode.values().stream().flatMap(List::stream).collect(Collectors.toSet());
        AbstractSnapshotVerificationTask.checkMissedMetadata(allMetas);
        IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr();
        synchronized (this) {
            opCtx0.stopFut = new IgniteFutureImpl<>(retFut.chain(f -> null));
        }
        if (log.isInfoEnabled()) {
            log.info("Starting snapshot preload operation to restore cache groups " + "[reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", caches=" + F.transform(opCtx0.dirs, FilePageStoreManager::cacheGroupName) + ']');
        }
        CompletableFuture<Void> metaFut = ctx.localNodeId().equals(opCtx0.opNodeId) ? CompletableFuture.runAsync(() -> {
            try {
                SnapshotMetadata meta = F.first(opCtx0.metasPerNode.get(opCtx0.opNodeId));
                File binDir = binaryWorkDir(snpMgr.snapshotLocalDir(opCtx0.snpName).getAbsolutePath(), meta.folderName());
                ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker);
            } catch (Throwable t) {
                log.error("Unable to perform metadata update operation for the cache groups restore process", t);
                opCtx0.errHnd.accept(t);
            }
        }, snpMgr.snapshotExecutorService()) : CompletableFuture.completedFuture(null);
        Map<String, GridAffinityAssignmentCache> affCache = new HashMap<>();
        for (StoredCacheData data : opCtx0.cfgs.values()) {
            affCache.computeIfAbsent(CU.cacheOrGroupName(data.config()), grp -> calculateAffinity(ctx, data.config(), opCtx0.discoCache));
        }
        Map<Integer, Set<PartitionRestoreFuture>> allParts = new HashMap<>();
        Map<Integer, Set<PartitionRestoreFuture>> rmtLoadParts = new HashMap<>();
        ClusterNode locNode = ctx.cache().context().localNode();
        List<SnapshotMetadata> locMetas = opCtx0.metasPerNode.get(locNode.id());
        // First preload everything from the local node.
        for (File dir : opCtx0.dirs) {
            String cacheOrGrpName = cacheGroupName(dir);
            int grpId = CU.cacheId(cacheOrGrpName);
            File tmpCacheDir = formatTmpDirName(dir);
            tmpCacheDir.mkdir();
            Set<PartitionRestoreFuture> leftParts;
            // Partitions contained in the snapshot.
            Set<Integer> availParts = new HashSet<>();
            for (SnapshotMetadata meta : allMetas) {
                Set<Integer> parts = meta.partitions().get(grpId);
                if (parts != null)
                    availParts.addAll(parts);
            }
            List<List<ClusterNode>> assignment = affCache.get(cacheOrGrpName).idealAssignment().assignment();
            Set<PartitionRestoreFuture> partFuts = availParts.stream().filter(p -> p != INDEX_PARTITION && assignment.get(p).contains(locNode)).map(p -> new PartitionRestoreFuture(p, opCtx0.processedParts)).collect(Collectors.toSet());
            allParts.put(grpId, partFuts);
            rmtLoadParts.put(grpId, leftParts = new HashSet<>(partFuts));
            if (leftParts.isEmpty())
                continue;
            SnapshotMetadata full = findMetadataWithSamePartitions(locMetas, grpId, leftParts.stream().map(p -> p.partId).collect(Collectors.toSet()));
            for (SnapshotMetadata meta : full == null ? locMetas : Collections.singleton(full)) {
                if (leftParts.isEmpty())
                    break;
                File snpCacheDir = new File(ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName), Paths.get(databaseRelativePath(meta.folderName()), dir.getName()).toString());
                leftParts.removeIf(partFut -> {
                    boolean doCopy = ofNullable(meta.partitions().get(grpId)).orElse(Collections.emptySet()).contains(partFut.partId);
                    if (doCopy) {
                        copyLocalAsync(ctx.cache().context().snapshotMgr(), opCtx0, snpCacheDir, tmpCacheDir, partFut);
                    }
                    return doCopy;
                });
                if (meta == full) {
                    assert leftParts.isEmpty() : leftParts;
                    if (log.isInfoEnabled()) {
                        log.info("The snapshot was taken on the same cluster topology. The index will be copied to " + "restoring cache group if necessary [reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", dir=" + dir.getName() + ']');
                    }
                    File idxFile = new File(snpCacheDir, FilePageStoreManager.getPartitionFileName(INDEX_PARTITION));
                    if (idxFile.exists()) {
                        PartitionRestoreFuture idxFut;
                        allParts.computeIfAbsent(grpId, g -> new HashSet<>()).add(idxFut = new PartitionRestoreFuture(INDEX_PARTITION, opCtx0.processedParts));
                        copyLocalAsync(ctx.cache().context().snapshotMgr(), opCtx0, snpCacheDir, tmpCacheDir, idxFut);
                    }
                }
            }
        }
        // Load other partitions from remote nodes.
        List<PartitionRestoreFuture> rmtAwaitParts = rmtLoadParts.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
        // This is necessary for sending only one partitions request per each cluster node.
        Map<UUID, Map<Integer, Set<Integer>>> snpAff = snapshotAffinity(opCtx0.metasPerNode.entrySet().stream().filter(e -> !e.getKey().equals(ctx.localNodeId())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), (grpId, partId) -> rmtLoadParts.get(grpId) != null && rmtLoadParts.get(grpId).remove(new PartitionRestoreFuture(partId, opCtx0.processedParts)));
        Map<Integer, File> grpToDir = opCtx0.dirs.stream().collect(Collectors.toMap(d -> CU.cacheId(FilePageStoreManager.cacheGroupName(d)), d -> d));
        try {
            if (log.isInfoEnabled() && !snpAff.isEmpty()) {
                log.info("Trying to request partitions from remote nodes " + "[reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", map=" + snpAff.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> partitionsMapToCompactString(e.getValue()))) + ']');
            }
            for (Map.Entry<UUID, Map<Integer, Set<Integer>>> m : snpAff.entrySet()) {
                ctx.cache().context().snapshotMgr().requestRemoteSnapshotFiles(m.getKey(), opCtx0.snpName, m.getValue(), opCtx0.stopChecker, (snpFile, t) -> {
                    if (opCtx0.stopChecker.getAsBoolean())
                        throw new IgniteInterruptedException("Snapshot remote operation request cancelled.");
                    if (t == null) {
                        int grpId = CU.cacheId(cacheGroupName(snpFile.getParentFile()));
                        int partId = partId(snpFile.getName());
                        PartitionRestoreFuture partFut = F.find(allParts.get(grpId), null, new IgnitePredicate<PartitionRestoreFuture>() {

                            @Override
                            public boolean apply(PartitionRestoreFuture f) {
                                return f.partId == partId;
                            }
                        });
                        assert partFut != null : snpFile.getAbsolutePath();
                        File tmpCacheDir = formatTmpDirName(grpToDir.get(grpId));
                        Path partFile = Paths.get(tmpCacheDir.getAbsolutePath(), snpFile.getName());
                        try {
                            Files.move(snpFile.toPath(), partFile);
                            partFut.complete(partFile);
                        } catch (Exception e) {
                            opCtx0.errHnd.accept(e);
                            completeListExceptionally(rmtAwaitParts, e);
                        }
                    } else {
                        opCtx0.errHnd.accept(t);
                        completeListExceptionally(rmtAwaitParts, t);
                    }
                });
            }
        } catch (IgniteCheckedException e) {
            opCtx0.errHnd.accept(e);
            completeListExceptionally(rmtAwaitParts, e);
        }
        List<PartitionRestoreFuture> allPartFuts = allParts.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
        int size = allPartFuts.size();
        opCtx0.totalParts = size;
        CompletableFuture.allOf(allPartFuts.toArray(new CompletableFuture[size])).runAfterBothAsync(metaFut, () -> {
            try {
                if (opCtx0.stopChecker.getAsBoolean())
                    throw new IgniteInterruptedException("The operation has been stopped on temporary directory switch.");
                for (File src : opCtx0.dirs) Files.move(formatTmpDirName(src).toPath(), src.toPath(), StandardCopyOption.ATOMIC_MOVE);
            } catch (IOException e) {
                throw new IgniteException(e);
            }
        }, snpMgr.snapshotExecutorService()).whenComplete((r, t) -> opCtx0.errHnd.accept(t)).whenComplete((res, t) -> {
            Throwable t0 = ofNullable(opCtx0.err.get()).orElse(t);
            if (t0 == null)
                retFut.onDone(true);
            else {
                log.error("Unable to restore cache group(s) from a snapshot " + "[reqId=" + opCtx.reqId + ", snapshot=" + opCtx.snpName + ']', t0);
                retFut.onDone(t0);
            }
        });
    } catch (Exception ex) {
        opCtx0.errHnd.accept(ex);
        return new GridFinishedFuture<>(ex);
    }
    return retFut;
}

Example 2 with FilePageStoreManager.cacheGroupName

use of org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName in project ignite by apache.

the class SnapshotRestoreProcess method prepareContext.

/**
 * @param req Request to prepare cache group restore from the snapshot.
 * @param metas Local snapshot metadatas.
 * @return Snapshot restore operation context.
 * @throws IgniteCheckedException If failed.
 */
private SnapshotRestoreContext prepareContext(SnapshotOperationRequest req, Collection<SnapshotMetadata> metas) throws IgniteCheckedException {
    if (opCtx != null) {
        throw new IgniteCheckedException(OP_REJECT_MSG + "The previous snapshot restore operation was not completed.");
    }
    GridCacheSharedContext<?, ?> cctx = ctx.cache().context();
    // Collection of baseline nodes that must survive and additional discovery data required for the affinity calculation.
    DiscoCache discoCache = ctx.discovery().discoCache();
    if (!F.transform(discoCache.aliveBaselineNodes(), F.node2id()).containsAll(req.nodes()))
        throw new IgniteCheckedException("Restore context cannot be inited since the required baseline nodes missed: " + discoCache);
    DiscoCache discoCache0 = discoCache.copy(discoCache.version(), null);
    if (F.isEmpty(metas))
        return new SnapshotRestoreContext(req, discoCache0, Collections.emptyMap());
    if (F.first(metas).pageSize() != cctx.database().pageSize()) {
        throw new IgniteCheckedException("Incompatible memory page size " + "[snapshotPageSize=" + F.first(metas).pageSize() + ", local=" + cctx.database().pageSize() + ", snapshot=" + req.snapshotName() + ", nodeId=" + cctx.localNodeId() + ']');
    }
    Map<String, StoredCacheData> cfgsByName = new HashMap<>();
    FilePageStoreManager pageStore = (FilePageStoreManager) cctx.pageStore();
    // Metastorage can be restored only manually by directly copying files.
    for (SnapshotMetadata meta : metas) {
        for (File snpCacheDir : cctx.snapshotMgr().snapshotCacheDirectories(req.snapshotName(), meta.folderName(), name -> !METASTORAGE_CACHE_NAME.equals(name))) {
            String grpName = FilePageStoreManager.cacheGroupName(snpCacheDir);
            if (!F.isEmpty(req.groups()) && !req.groups().contains(grpName))
                continue;
            File cacheDir = pageStore.cacheWorkDir(snpCacheDir.getName().startsWith(CACHE_GRP_DIR_PREFIX), grpName);
            if (cacheDir.exists()) {
                if (!cacheDir.isDirectory()) {
                    throw new IgniteCheckedException("Unable to restore cache group, file with required directory " + "name already exists [group=" + grpName + ", file=" + cacheDir + ']');
                }
                if (cacheDir.list().length > 0) {
                    throw new IgniteCheckedException("Unable to restore cache group - directory is not empty. " + "Cache group should be destroyed manually before perform restore operation " + "[group=" + grpName + ", dir=" + cacheDir + ']');
                }
                if (!cacheDir.delete()) {
                    throw new IgniteCheckedException("Unable to remove empty cache directory " + "[group=" + grpName + ", dir=" + cacheDir + ']');
                }
            }
            File tmpCacheDir = formatTmpDirName(cacheDir);
            if (tmpCacheDir.exists()) {
                throw new IgniteCheckedException("Unable to restore cache group, temp directory already exists " + "[group=" + grpName + ", dir=" + tmpCacheDir + ']');
            }
            pageStore.readCacheConfigurations(snpCacheDir, cfgsByName);
        }
    }
    Map<Integer, StoredCacheData> cfgsById = cfgsByName.values().stream().collect(Collectors.toMap(v -> CU.cacheId(v.config().getName()), v -> v));
    return new SnapshotRestoreContext(req, discoCache0, cfgsById);
}