Examples with StoredCacheData org.apache.ignite.internal.processors.cache.StoredCacheData used on opensource projects

Example 1 with StoredCacheData

use of org.apache.ignite.internal.processors.cache.StoredCacheData in project ignite by apache.

the class GridClusterStateProcessor method changeGlobalState0.

/**
 */
private IgniteInternalFuture<?> changeGlobalState0(final boolean activate, BaselineTopology blt, boolean forceChangeBaselineTopology) {
    if (ctx.isDaemon() || ctx.clientNode()) {
        GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
        sendComputeChangeGlobalState(activate, blt, forceChangeBaselineTopology, fut);
        return fut;
    }
    if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) + " cluster (must invoke the method outside of an active transaction)."));
    }
    DiscoveryDataClusterState curState = globalState;
    if (!curState.transition() && curState.active() == activate && BaselineTopology.equals(curState.baselineTopology(), blt))
        return new GridFinishedFuture<>();
    GridChangeGlobalStateFuture startedFut = null;
    GridChangeGlobalStateFuture fut = stateChangeFut.get();
    while (fut == null || fut.isDone()) {
        fut = new GridChangeGlobalStateFuture(UUID.randomUUID(), activate, ctx);
        if (stateChangeFut.compareAndSet(null, fut)) {
            startedFut = fut;
            break;
        } else
            fut = stateChangeFut.get();
    }
    if (startedFut == null) {
        if (fut.activate != activate) {
            return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) + ", because another state change operation is currently in progress: " + prettyStr(fut.activate)));
        } else
            return fut;
    }
    List<StoredCacheData> storedCfgs = null;
    if (activate && CU.isPersistenceEnabled(ctx.config())) {
        try {
            Map<String, StoredCacheData> cfgs = ctx.cache().context().pageStore().readCacheConfigurations();
            if (!F.isEmpty(cfgs))
                storedCfgs = new ArrayList<>(cfgs.values());
        } catch (IgniteCheckedException e) {
            U.error(log, "Failed to read stored cache configurations: " + e, e);
            startedFut.onDone(e);
            return startedFut;
        }
    }
    ChangeGlobalStateMessage msg = new ChangeGlobalStateMessage(startedFut.requestId, ctx.localNodeId(), storedCfgs, activate, blt, forceChangeBaselineTopology, System.currentTimeMillis());
    try {
        if (log.isInfoEnabled())
            U.log(log, "Sending " + prettyStr(activate) + " request with BaselineTopology " + blt);
        ctx.discovery().sendCustomEvent(msg);
        if (ctx.isStopping())
            startedFut.onDone(new IgniteCheckedException("Failed to execute " + prettyStr(activate) + " request, " + "node is stopping."));
    } catch (IgniteCheckedException e) {
        U.error(log, "Failed to send global state change request: " + activate, e);
        startedFut.onDone(e);
    }
    return wrapStateChangeFuture(startedFut, msg);
}

Example 2 with StoredCacheData

use of org.apache.ignite.internal.processors.cache.StoredCacheData in project ignite by apache.

the class FilePageStoreManager method readCacheGroupCaches.

/**
 * @param grpDir Group directory.
 * @param ccfgs Cache configurations.
 * @throws IgniteCheckedException If failed.
 */
private void readCacheGroupCaches(File grpDir, Map<String, StoredCacheData> ccfgs) throws IgniteCheckedException {
    File[] files = grpDir.listFiles();
    if (files == null)
        return;
    for (File file : files) {
        if (!file.isDirectory() && file.getName().endsWith(CACHE_DATA_FILENAME) && file.length() > 0) {
            StoredCacheData cacheData = readCacheData(file);
            String cacheName = cacheData.config().getName();
            if (!ccfgs.containsKey(cacheName))
                ccfgs.put(cacheName, cacheData);
            else {
                U.warn(log, "Cache with name=" + cacheName + " is already registered, skipping config file " + file.getName() + " in group directory " + grpDir.getName());
            }
        }
    }
}

Example 3 with StoredCacheData

use of org.apache.ignite.internal.processors.cache.StoredCacheData in project ignite by apache.

the class SnapshotRestoreProcess method isRestoring.

/**
 * Check if the cache or group with the specified name is currently being restored from the snapshot.
 * @param opCtx Restoring context.
 * @param ccfg Cache configuration.
 * @return {@code True} if the cache or group with the specified name is currently being restored.
 */
private boolean isRestoring(CacheConfiguration<?, ?> ccfg, @Nullable SnapshotRestoreContext opCtx) {
    assert ccfg != null;
    if (opCtx == null)
        return false;
    Map<Integer, StoredCacheData> cacheCfgs = opCtx.cfgs;
    String cacheName = ccfg.getName();
    String grpName = ccfg.getGroupName();
    int cacheId = CU.cacheId(cacheName);
    if (cacheCfgs.containsKey(cacheId))
        return true;
    for (File grpDir : opCtx.dirs) {
        String locGrpName = FilePageStoreManager.cacheGroupName(grpDir);
        if (grpName != null) {
            if (cacheName.equals(locGrpName))
                return true;
            if (CU.cacheId(locGrpName) == CU.cacheId(grpName))
                return true;
        } else if (CU.cacheId(locGrpName) == cacheId)
            return true;
    }
    return false;
}

Example 4 with StoredCacheData

use of org.apache.ignite.internal.processors.cache.StoredCacheData in project ignite by apache.

the class SnapshotRestoreProcess method prepare.

/**
 * @param req Request to prepare cache group restore from the snapshot.
 * @return Result future.
 */
private IgniteInternalFuture<SnapshotRestoreOperationResponse> prepare(SnapshotOperationRequest req) {
    if (ctx.clientNode())
        return new GridFinishedFuture<>();
    try {
        DiscoveryDataClusterState state = ctx.state().clusterState();
        IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr();
        if (state.state() != ClusterState.ACTIVE || state.transition())
            throw new IgniteCheckedException(OP_REJECT_MSG + "The cluster should be active.");
        if (snpMgr.isSnapshotCreating())
            throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress.");
        if (ctx.encryption().isMasterKeyChangeInProgress()) {
            return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " + "process is not finished yet."));
        }
        if (ctx.encryption().reencryptionInProgress()) {
            return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Caches re-encryption " + "process is not finished yet."));
        }
        for (UUID nodeId : req.nodes()) {
            ClusterNode node = ctx.discovery().node(nodeId);
            if (node == null || !CU.baselineNode(node, state) || !ctx.discovery().alive(node)) {
                throw new IgniteCheckedException(OP_REJECT_MSG + "Required node has left the cluster [nodeId-" + nodeId + ']');
            }
        }
        if (log.isInfoEnabled()) {
            log.info("Starting local snapshot prepare restore operation" + " [reqId=" + req.requestId() + ", snapshot=" + req.snapshotName() + ", caches=" + req.groups() + ']');
        }
        List<SnapshotMetadata> locMetas = snpMgr.readSnapshotMetadatas(req.snapshotName());
        SnapshotRestoreContext opCtx0 = prepareContext(req, locMetas);
        synchronized (this) {
            lastOpCtx = opCtx = opCtx0;
            ClusterSnapshotFuture fut0 = fut;
            if (fut0 != null)
                opCtx0.errHnd.accept(fut0.interruptEx);
        }
        // Ensure that shared cache groups has no conflicts.
        for (StoredCacheData cfg : opCtx0.cfgs.values()) {
            ensureCacheAbsent(cfg.config().getName());
            if (!F.isEmpty(cfg.config().getGroupName()))
                ensureCacheAbsent(cfg.config().getGroupName());
        }
        if (ctx.isStopping())
            throw new NodeStoppingException("The node is stopping: " + ctx.localNodeId());
        return new GridFinishedFuture<>(new SnapshotRestoreOperationResponse(opCtx0.cfgs.values(), locMetas));
    } catch (IgniteIllegalStateException | IgniteCheckedException | RejectedExecutionException e) {
        log.error("Unable to restore cache group(s) from the snapshot " + "[reqId=" + req.requestId() + ", snapshot=" + req.snapshotName() + ']', e);
        return new GridFinishedFuture<>(e);
    }
}

Example 5 with StoredCacheData

use of org.apache.ignite.internal.processors.cache.StoredCacheData in project ignite by apache.

the class SnapshotRestoreProcess method preload.

/**
 * @param reqId Request id.
 * @return Future which will be completed when the preload ends.
 */
private IgniteInternalFuture<Boolean> preload(UUID reqId) {
    if (ctx.clientNode())
        return new GridFinishedFuture<>();
    SnapshotRestoreContext opCtx0 = opCtx;
    GridFutureAdapter<Boolean> retFut = new GridFutureAdapter<>();
    if (opCtx0 == null)
        return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot restore process has incorrect restore state: " + reqId));
    if (opCtx0.dirs.isEmpty())
        return new GridFinishedFuture<>();
    try {
        if (ctx.isStopping())
            throw new NodeStoppingException("Node is stopping: " + ctx.localNodeId());
        Set<SnapshotMetadata> allMetas = opCtx0.metasPerNode.values().stream().flatMap(List::stream).collect(Collectors.toSet());
        AbstractSnapshotVerificationTask.checkMissedMetadata(allMetas);
        IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr();
        synchronized (this) {
            opCtx0.stopFut = new IgniteFutureImpl<>(retFut.chain(f -> null));
        }
        if (log.isInfoEnabled()) {
            log.info("Starting snapshot preload operation to restore cache groups " + "[reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", caches=" + F.transform(opCtx0.dirs, FilePageStoreManager::cacheGroupName) + ']');
        }
        CompletableFuture<Void> metaFut = ctx.localNodeId().equals(opCtx0.opNodeId) ? CompletableFuture.runAsync(() -> {
            try {
                SnapshotMetadata meta = F.first(opCtx0.metasPerNode.get(opCtx0.opNodeId));
                File binDir = binaryWorkDir(snpMgr.snapshotLocalDir(opCtx0.snpName).getAbsolutePath(), meta.folderName());
                ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker);
            } catch (Throwable t) {
                log.error("Unable to perform metadata update operation for the cache groups restore process", t);
                opCtx0.errHnd.accept(t);
            }
        }, snpMgr.snapshotExecutorService()) : CompletableFuture.completedFuture(null);
        Map<String, GridAffinityAssignmentCache> affCache = new HashMap<>();
        for (StoredCacheData data : opCtx0.cfgs.values()) {
            affCache.computeIfAbsent(CU.cacheOrGroupName(data.config()), grp -> calculateAffinity(ctx, data.config(), opCtx0.discoCache));
        }
        Map<Integer, Set<PartitionRestoreFuture>> allParts = new HashMap<>();
        Map<Integer, Set<PartitionRestoreFuture>> rmtLoadParts = new HashMap<>();
        ClusterNode locNode = ctx.cache().context().localNode();
        List<SnapshotMetadata> locMetas = opCtx0.metasPerNode.get(locNode.id());
        // First preload everything from the local node.
        for (File dir : opCtx0.dirs) {
            String cacheOrGrpName = cacheGroupName(dir);
            int grpId = CU.cacheId(cacheOrGrpName);
            File tmpCacheDir = formatTmpDirName(dir);
            tmpCacheDir.mkdir();
            Set<PartitionRestoreFuture> leftParts;
            // Partitions contained in the snapshot.
            Set<Integer> availParts = new HashSet<>();
            for (SnapshotMetadata meta : allMetas) {
                Set<Integer> parts = meta.partitions().get(grpId);
                if (parts != null)
                    availParts.addAll(parts);
            }
            List<List<ClusterNode>> assignment = affCache.get(cacheOrGrpName).idealAssignment().assignment();
            Set<PartitionRestoreFuture> partFuts = availParts.stream().filter(p -> p != INDEX_PARTITION && assignment.get(p).contains(locNode)).map(p -> new PartitionRestoreFuture(p, opCtx0.processedParts)).collect(Collectors.toSet());
            allParts.put(grpId, partFuts);
            rmtLoadParts.put(grpId, leftParts = new HashSet<>(partFuts));
            if (leftParts.isEmpty())
                continue;
            SnapshotMetadata full = findMetadataWithSamePartitions(locMetas, grpId, leftParts.stream().map(p -> p.partId).collect(Collectors.toSet()));
            for (SnapshotMetadata meta : full == null ? locMetas : Collections.singleton(full)) {
                if (leftParts.isEmpty())
                    break;
                File snpCacheDir = new File(ctx.cache().context().snapshotMgr().snapshotLocalDir(opCtx0.snpName), Paths.get(databaseRelativePath(meta.folderName()), dir.getName()).toString());
                leftParts.removeIf(partFut -> {
                    boolean doCopy = ofNullable(meta.partitions().get(grpId)).orElse(Collections.emptySet()).contains(partFut.partId);
                    if (doCopy) {
                        copyLocalAsync(ctx.cache().context().snapshotMgr(), opCtx0, snpCacheDir, tmpCacheDir, partFut);
                    }
                    return doCopy;
                });
                if (meta == full) {
                    assert leftParts.isEmpty() : leftParts;
                    if (log.isInfoEnabled()) {
                        log.info("The snapshot was taken on the same cluster topology. The index will be copied to " + "restoring cache group if necessary [reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", dir=" + dir.getName() + ']');
                    }
                    File idxFile = new File(snpCacheDir, FilePageStoreManager.getPartitionFileName(INDEX_PARTITION));
                    if (idxFile.exists()) {
                        PartitionRestoreFuture idxFut;
                        allParts.computeIfAbsent(grpId, g -> new HashSet<>()).add(idxFut = new PartitionRestoreFuture(INDEX_PARTITION, opCtx0.processedParts));
                        copyLocalAsync(ctx.cache().context().snapshotMgr(), opCtx0, snpCacheDir, tmpCacheDir, idxFut);
                    }
                }
            }
        }
        // Load other partitions from remote nodes.
        List<PartitionRestoreFuture> rmtAwaitParts = rmtLoadParts.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
        // This is necessary for sending only one partitions request per each cluster node.
        Map<UUID, Map<Integer, Set<Integer>>> snpAff = snapshotAffinity(opCtx0.metasPerNode.entrySet().stream().filter(e -> !e.getKey().equals(ctx.localNodeId())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)), (grpId, partId) -> rmtLoadParts.get(grpId) != null && rmtLoadParts.get(grpId).remove(new PartitionRestoreFuture(partId, opCtx0.processedParts)));
        Map<Integer, File> grpToDir = opCtx0.dirs.stream().collect(Collectors.toMap(d -> CU.cacheId(FilePageStoreManager.cacheGroupName(d)), d -> d));
        try {
            if (log.isInfoEnabled() && !snpAff.isEmpty()) {
                log.info("Trying to request partitions from remote nodes " + "[reqId=" + reqId + ", snapshot=" + opCtx0.snpName + ", map=" + snpAff.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> partitionsMapToCompactString(e.getValue()))) + ']');
            }
            for (Map.Entry<UUID, Map<Integer, Set<Integer>>> m : snpAff.entrySet()) {
                ctx.cache().context().snapshotMgr().requestRemoteSnapshotFiles(m.getKey(), opCtx0.snpName, m.getValue(), opCtx0.stopChecker, (snpFile, t) -> {
                    if (opCtx0.stopChecker.getAsBoolean())
                        throw new IgniteInterruptedException("Snapshot remote operation request cancelled.");
                    if (t == null) {
                        int grpId = CU.cacheId(cacheGroupName(snpFile.getParentFile()));
                        int partId = partId(snpFile.getName());
                        PartitionRestoreFuture partFut = F.find(allParts.get(grpId), null, new IgnitePredicate<PartitionRestoreFuture>() {

                            @Override
                            public boolean apply(PartitionRestoreFuture f) {
                                return f.partId == partId;
                            }
                        });
                        assert partFut != null : snpFile.getAbsolutePath();
                        File tmpCacheDir = formatTmpDirName(grpToDir.get(grpId));
                        Path partFile = Paths.get(tmpCacheDir.getAbsolutePath(), snpFile.getName());
                        try {
                            Files.move(snpFile.toPath(), partFile);
                            partFut.complete(partFile);
                        } catch (Exception e) {
                            opCtx0.errHnd.accept(e);
                            completeListExceptionally(rmtAwaitParts, e);
                        }
                    } else {
                        opCtx0.errHnd.accept(t);
                        completeListExceptionally(rmtAwaitParts, t);
                    }
                });
            }
        } catch (IgniteCheckedException e) {
            opCtx0.errHnd.accept(e);
            completeListExceptionally(rmtAwaitParts, e);
        }
        List<PartitionRestoreFuture> allPartFuts = allParts.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
        int size = allPartFuts.size();
        opCtx0.totalParts = size;
        CompletableFuture.allOf(allPartFuts.toArray(new CompletableFuture[size])).runAfterBothAsync(metaFut, () -> {
            try {
                if (opCtx0.stopChecker.getAsBoolean())
                    throw new IgniteInterruptedException("The operation has been stopped on temporary directory switch.");
                for (File src : opCtx0.dirs) Files.move(formatTmpDirName(src).toPath(), src.toPath(), StandardCopyOption.ATOMIC_MOVE);
            } catch (IOException e) {
                throw new IgniteException(e);
            }
        }, snpMgr.snapshotExecutorService()).whenComplete((r, t) -> opCtx0.errHnd.accept(t)).whenComplete((res, t) -> {
            Throwable t0 = ofNullable(opCtx0.err.get()).orElse(t);
            if (t0 == null)
                retFut.onDone(true);
            else {
                log.error("Unable to restore cache group(s) from a snapshot " + "[reqId=" + opCtx.reqId + ", snapshot=" + opCtx.snpName + ']', t0);
                retFut.onDone(t0);
            }
        });
    } catch (Exception ex) {
        opCtx0.errHnd.accept(ex);
        return new GridFinishedFuture<>(ex);
    }
    return retFut;
}