Examples with GroupPartitionId org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId used on opensource projects

Example 6 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class GridCacheProcessor method toStringTopProcessingPartitions.

/**
 * Creation of a string representation of the top (descending) partitions, the processing of which took the most time.
 *
 * @param top Top (ascending) processed partitions.
 * @param groups Cache group contexts.
 * @return String representation.
 */
static String toStringTopProcessingPartitions(SortedSet<T3<Long, Long, GroupPartitionId>> top, Collection<CacheGroupContext> groups) {
    if (top.isEmpty())
        return "[]";
    StringJoiner sj0 = new StringJoiner(", ", "[", "]");
    TreeMap<Long, List<GroupPartitionId>> top0 = top.stream().collect(groupingBy(T3::get1, TreeMap::new, mapping(T3::get3, toList())));
    for (Map.Entry<Long, List<GroupPartitionId>> e0 : top0.descendingMap().entrySet()) {
        Map<Integer, List<GroupPartitionId>> byCacheGrpId = e0.getValue().stream().collect(groupingBy(GroupPartitionId::getGroupId));
        StringJoiner sj1 = new StringJoiner(", ", "[", "]");
        for (Map.Entry<Integer, List<GroupPartitionId>> e1 : byCacheGrpId.entrySet()) {
            @Nullable CacheGroupContext grp = groups.stream().filter(g -> g.groupId() == e1.getKey()).findAny().orElse(null);
            String parts = e1.getValue().stream().map(GroupPartitionId::getPartitionId).sorted().map(p -> grp == null ? p.toString() : p + ":" + grp.topology().localPartition(p).fullSize()).collect(Collectors.joining(", ", "[", "]"));
            sj1.add("[grp=" + (grp == null ? e1.getKey() : grp.cacheOrGroupName()) + ", part=" + parts + ']');
        }
        sj0.add("[time=" + U.humanReadableDuration(e0.getKey()) + ' ' + sj1.toString() + ']');
    }
    return sj0.toString();
}

Example 7 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class SnapshotFutureTask method addPartitionWriters.

/**
 * @param grpId Cache group id.
 * @param parts Set of partitions to be processed.
 * @param dirName Directory name to init.
 * @throws IgniteCheckedException If fails.
 */
private void addPartitionWriters(int grpId, Set<Integer> parts, String dirName) throws IgniteCheckedException {
    Integer encGrpId = cctx.cache().isEncrypted(grpId) ? grpId : null;
    for (int partId : parts) {
        GroupPartitionId pair = new GroupPartitionId(grpId, partId);
        PageStore store = pageStore.getStore(grpId, partId);
        partDeltaWriters.put(pair, new PageStoreSerialWriter(store, partDeltaFile(cacheWorkDir(tmpConsIdDir, dirName), partId), encGrpId));
        partFileLengths.put(pair, store.size());
    }
}

Example 8 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class IgniteSnapshotManagerSelfTest method testLocalSnapshotOnCacheStopped.

/**
 * @throws Exception If fails.
 */
@Test(expected = IgniteCheckedException.class)
public void testLocalSnapshotOnCacheStopped() throws Exception {
    IgniteEx ig = startGridWithCache(dfltCacheCfg, CACHE_KEYS_RANGE);
    startGrid(1);
    ig.cluster().state(ClusterState.ACTIVE);
    awaitPartitionMapExchange();
    GridCacheSharedContext<?, ?> cctx0 = ig.context().cache().context();
    IgniteSnapshotManager mgr = snp(ig);
    CountDownLatch cpLatch = new CountDownLatch(1);
    IgniteInternalFuture<?> snpFut = startLocalSnapshotTask(cctx0, SNAPSHOT_NAME, F.asMap(CU.cacheId(DEFAULT_CACHE_NAME), null), encryption, new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(SNAPSHOT_NAME)) {

        @Override
        public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
            try {
                U.await(cpLatch);
                delegate.sendPart0(part, cacheDirName, pair, length);
            } catch (IgniteInterruptedCheckedException e) {
                throw new IgniteException(e);
            }
        }
    });
    IgniteCache<?, ?> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME);
    cache.destroy();
    cpLatch.countDown();
    snpFut.get(5_000, TimeUnit.MILLISECONDS);
}

Example 9 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class IgniteSnapshotRestoreFromRemoteTest method testSnapshotCachesStoppedIfLoadingFailOnRemote.

/**
 * @throws Exception If failed.
 */
@Test
public void testSnapshotCachesStoppedIfLoadingFailOnRemote() throws Exception {
    IgniteEx scc = startDedicatedGrids(SECOND_CLUSTER_PREFIX, 2);
    scc.cluster().state(ClusterState.ACTIVE);
    copyAndShuffle(snpParts, G.allGrids());
    grid(0).cache(DEFAULT_CACHE_NAME).destroy();
    IgniteSnapshotManager mgr = snp(grid(1));
    mgr.remoteSnapshotSenderFactory(new BiFunction<String, UUID, SnapshotSender>() {

        @Override
        public SnapshotSender apply(String s, UUID uuid) {
            return new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.remoteSnapshotSenderFactory(s, uuid)) {

                @Override
                public void sendPart0(File part, String cacheDirName, GroupPartitionId pair, Long length) {
                    if (partId(part.getName()) > 0)
                        throw new IgniteException("Test exception. Uploading partition file failed: " + pair);
                    super.sendPart0(part, cacheDirName, pair, length);
                }
            };
        }
    });
    IgniteFuture<?> fut = grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, null);
    GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteException.class, "Test exception. Uploading partition file failed");
    assertNull(scc.cache(DEFAULT_CACHE_NAME));
    ensureCacheAbsent(dfltCacheCfg);
}

Example 10 with GroupPartitionId

use of org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId in project ignite by apache.

the class IgniteSnapshotManager method initLocalSnapshotStartStage.

/**
 * @param req Request on snapshot creation.
 * @return Future which will be completed when a snapshot has been started.
 */
private IgniteInternalFuture<SnapshotOperationResponse> initLocalSnapshotStartStage(SnapshotOperationRequest req) {
    if (cctx.kernalContext().clientNode() || !CU.baselineNode(cctx.localNode(), cctx.kernalContext().state().clusterState()))
        return new GridFinishedFuture<>();
    // so it is safe to set new snapshot task inside this method without synchronization.
    if (clusterSnpReq != null) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. " + "Another snapshot operation in progress [req=" + req + ", curr=" + clusterSnpReq + ']'));
    }
    Set<UUID> leftNodes = new HashSet<>(req.nodes());
    leftNodes.removeAll(F.viewReadOnly(cctx.discovery().serverNodes(AffinityTopologyVersion.NONE), F.node2id()));
    if (!leftNodes.isEmpty()) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Some of baseline nodes left the cluster " + "prior to snapshot operation start: " + leftNodes));
    }
    if (!cctx.localNode().isClient() && cctx.kernalContext().encryption().isMasterKeyChangeInProgress()) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Master " + "key changing process is not finished yet."));
    }
    if (!cctx.localNode().isClient() && cctx.kernalContext().encryption().reencryptionInProgress()) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Caches " + "re-encryption process is not finished yet."));
    }
    List<Integer> grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId));
    Set<Integer> leftGrps = new HashSet<>(grpIds);
    leftGrps.removeAll(cctx.cache().cacheGroupDescriptors().keySet());
    boolean withMetaStorage = leftGrps.remove(METASTORAGE_CACHE_ID);
    if (!leftGrps.isEmpty()) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Some of requested cache groups doesn't exist " + "on the local node [missed=" + leftGrps + ", nodeId=" + cctx.localNodeId() + ']'));
    }
    Map<Integer, Set<Integer>> parts = new HashMap<>();
    // Cache group context may be 'null' on some nodes e.g. a node filter is set.
    for (Integer grpId : grpIds) {
        if (cctx.cache().cacheGroup(grpId) == null)
            continue;
        parts.put(grpId, null);
    }
    IgniteInternalFuture<?> task0;
    if (parts.isEmpty() && !withMetaStorage)
        task0 = new GridFinishedFuture<>(Collections.emptySet());
    else {
        task0 = registerSnapshotTask(req.snapshotName(), req.operationalNodeId(), parts, withMetaStorage, locSndrFactory.apply(req.snapshotName()));
        if (withMetaStorage && task0 instanceof SnapshotFutureTask) {
            ((DistributedMetaStorageImpl) cctx.kernalContext().distributedMetastorage()).suspend(((SnapshotFutureTask) task0).started());
        }
        clusterSnpReq = req;
    }
    return task0.chain(fut -> {
        if (fut.error() != null)
            throw F.wrap(fut.error());
        try {
            Set<String> blts = req.nodes().stream().map(n -> cctx.discovery().node(n).consistentId().toString()).collect(Collectors.toSet());
            File smf = new File(snapshotLocalDir(req.snapshotName()), snapshotMetaFileName(cctx.localNode().consistentId().toString()));
            if (smf.exists())
                throw new GridClosureException(new IgniteException("Snapshot metafile must not exist: " + smf.getAbsolutePath()));
            smf.getParentFile().mkdirs();
            SnapshotMetadata meta = new SnapshotMetadata(req.requestId(), req.snapshotName(), cctx.localNode().consistentId().toString(), pdsSettings.folderName(), cctx.gridConfig().getDataStorageConfiguration().getPageSize(), grpIds, blts, (Set<GroupPartitionId>) fut.result());
            try (OutputStream out = new BufferedOutputStream(new FileOutputStream(smf))) {
                U.marshal(marsh, meta, out);
                log.info("Snapshot metafile has been created: " + smf.getAbsolutePath());
            }
            SnapshotHandlerContext ctx = new SnapshotHandlerContext(meta, req.groups(), cctx.localNode());
            return new SnapshotOperationResponse(handlers.invokeAll(SnapshotHandlerType.CREATE, ctx));
        } catch (IOException | IgniteCheckedException e) {
            throw F.wrap(e);
        }
    });
}