Search in sources :

Example 1 with PartitionKeyV2

use of org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2 in project ignite by apache.

the class SnapshotPartitionsVerifyHandler method invoke.

/**
 * {@inheritDoc}
 */
@Override
public Map<PartitionKeyV2, PartitionHashRecordV2> invoke(SnapshotHandlerContext opCtx) throws IgniteCheckedException {
    SnapshotMetadata meta = opCtx.metadata();
    Set<Integer> grps = F.isEmpty(opCtx.groups()) ? new HashSet<>(meta.partitions().keySet()) : opCtx.groups().stream().map(CU::cacheId).collect(Collectors.toSet());
    Set<File> partFiles = new HashSet<>();
    IgniteSnapshotManager snpMgr = cctx.snapshotMgr();
    for (File dir : snpMgr.snapshotCacheDirectories(meta.snapshotName(), meta.folderName())) {
        int grpId = CU.cacheId(cacheGroupName(dir));
        if (!grps.remove(grpId))
            continue;
        Set<Integer> parts = meta.partitions().get(grpId) == null ? Collections.emptySet() : new HashSet<>(meta.partitions().get(grpId));
        for (File part : cachePartitionFiles(dir)) {
            int partId = partId(part.getName());
            if (!parts.remove(partId))
                continue;
            partFiles.add(part);
        }
        if (!parts.isEmpty()) {
            throw new IgniteException("Snapshot data doesn't contain required cache group partition " + "[grpId=" + grpId + ", snpName=" + meta.snapshotName() + ", consId=" + meta.consistentId() + ", missed=" + parts + ", meta=" + meta + ']');
        }
    }
    if (!grps.isEmpty()) {
        throw new IgniteException("Snapshot data doesn't contain required cache groups " + "[grps=" + grps + ", snpName=" + meta.snapshotName() + ", consId=" + meta.consistentId() + ", meta=" + meta + ']');
    }
    Map<PartitionKeyV2, PartitionHashRecordV2> res = new ConcurrentHashMap<>();
    ThreadLocal<ByteBuffer> buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()).order(ByteOrder.nativeOrder()));
    GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(meta.snapshotName(), meta.folderName());
    for (GridComponent comp : snpCtx) comp.start();
    try {
        U.doInParallel(snpMgr.snapshotExecutorService(), partFiles, part -> {
            String grpName = cacheGroupName(part.getParentFile());
            int grpId = CU.cacheId(grpName);
            int partId = partId(part.getName());
            FilePageStoreManager storeMgr = (FilePageStoreManager) cctx.pageStore();
            try (FilePageStore pageStore = (FilePageStore) storeMgr.getPageStoreFactory(grpId, false).createPageStore(getTypeByPartId(partId), part::toPath, val -> {
            })) {
                if (partId == INDEX_PARTITION) {
                    checkPartitionsPageCrcSum(() -> pageStore, INDEX_PARTITION, FLAG_IDX);
                    return null;
                }
                if (grpId == MetaStorage.METASTORAGE_CACHE_ID) {
                    checkPartitionsPageCrcSum(() -> pageStore, partId, FLAG_DATA);
                    return null;
                }
                ByteBuffer pageBuff = buff.get();
                pageBuff.clear();
                pageStore.read(0, pageBuff, true);
                long pageAddr = GridUnsafe.bufferAddress(pageBuff);
                PagePartitionMetaIO io = PageIO.getPageIO(pageBuff);
                GridDhtPartitionState partState = fromOrdinal(io.getPartitionState(pageAddr));
                if (partState != OWNING) {
                    throw new IgniteCheckedException("Snapshot partitions must be in the OWNING " + "state only: " + partState);
                }
                long updateCntr = io.getUpdateCounter(pageAddr);
                long size = io.getSize(pageAddr);
                if (log.isDebugEnabled()) {
                    log.debug("Partition [grpId=" + grpId + ", id=" + partId + ", counter=" + updateCntr + ", size=" + size + "]");
                }
                // Snapshot partitions must always be in OWNING state.
                // There is no `primary` partitions for snapshot.
                PartitionKeyV2 key = new PartitionKeyV2(grpId, partId, grpName);
                PartitionHashRecordV2 hash = calculatePartitionHash(key, updateCntr, meta.consistentId(), GridDhtPartitionState.OWNING, false, size, snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore));
                assert hash != null : "OWNING must have hash: " + key;
                res.put(key, hash);
            } catch (IOException e) {
                throw new IgniteCheckedException(e);
            }
            return null;
        });
    } catch (Throwable t) {
        log.error("Error executing handler: ", t);
        throw t;
    } finally {
        for (GridComponent comp : snpCtx) comp.stop(true);
    }
    return res;
}
Also used : FLAG_DATA(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA) PartitionHashRecordV2(org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2) IdleVerifyUtility.calculatePartitionHash(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) IgniteLogger(org.apache.ignite.IgniteLogger) ByteBuffer(java.nio.ByteBuffer) IdleVerifyUtility.checkPartitionsPageCrcSum(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) HashSet(java.util.HashSet) ClusterNode(org.apache.ignite.cluster.ClusterNode) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) GridDhtPartitionState.fromOrdinal(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal) Map(java.util.Map) PageIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO) FilePageStoreManager.cachePartitionFiles(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) GridStringBuilder(org.apache.ignite.internal.util.GridStringBuilder) F(org.apache.ignite.internal.util.typedef.F) FilePageStoreManager.cacheGroupName(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) GridComponent(org.apache.ignite.internal.GridComponent) Set(java.util.Set) GridUnsafe(org.apache.ignite.internal.util.GridUnsafe) IOException(java.io.IOException) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) MetaStorage(org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) Collectors(java.util.stream.Collectors) File(java.io.File) FilePageStoreManager.partId(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) ByteOrder(java.nio.ByteOrder) List(java.util.List) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) PartitionKeyV2(org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2) CU(org.apache.ignite.internal.util.typedef.internal.CU) GroupPartitionId.getTypeByPartId(org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId) Collections(java.util.Collections) IdleVerifyResultV2(org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) GridKernalContext(org.apache.ignite.internal.GridKernalContext) GridComponent(org.apache.ignite.internal.GridComponent) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) PartitionKeyV2(org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashSet(java.util.HashSet) IOException(java.io.IOException) ByteBuffer(java.nio.ByteBuffer) CU(org.apache.ignite.internal.util.typedef.internal.CU) PartitionHashRecordV2(org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2) GridDhtPartitionState(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState) PagePartitionMetaIO(org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO) File(java.io.File)

Example 2 with PartitionKeyV2

use of org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2 in project ignite by apache.

the class SnapshotPartitionsVerifyHandler method complete.

/**
 * {@inheritDoc}
 */
@Override
public void complete(String name, Collection<SnapshotHandlerResult<Map<PartitionKeyV2, PartitionHashRecordV2>>> results) throws IgniteCheckedException {
    Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new HashMap<>();
    Map<ClusterNode, Exception> errs = new HashMap<>();
    for (SnapshotHandlerResult<Map<PartitionKeyV2, PartitionHashRecordV2>> res : results) {
        if (res.error() != null) {
            errs.put(res.node(), res.error());
            continue;
        }
        for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> entry : res.data().entrySet()) clusterHashes.computeIfAbsent(entry.getKey(), v -> new ArrayList<>()).add(entry.getValue());
    }
    IdleVerifyResultV2 verifyResult = new IdleVerifyResultV2(clusterHashes, errs);
    if (errs.isEmpty() && !verifyResult.hasConflicts())
        return;
    GridStringBuilder buf = new GridStringBuilder();
    verifyResult.print(buf::a, true);
    throw new IgniteCheckedException(buf.toString());
}
Also used : ClusterNode(org.apache.ignite.cluster.ClusterNode) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) GridStringBuilder(org.apache.ignite.internal.util.GridStringBuilder) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) IOException(java.io.IOException) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) PartitionKeyV2(org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2) PartitionHashRecordV2(org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2) ArrayList(java.util.ArrayList) List(java.util.List) IdleVerifyResultV2(org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 3 with PartitionKeyV2

use of org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2 in project ignite by apache.

the class IgniteClusterSnapshotCheckTest method saveHashes.

/**
 * @param cls Class of running task.
 * @param results Results of compute.
 */
private void saveHashes(Class<?> cls, List<ComputeJobResult> results) {
    Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashes = new HashMap<>();
    for (ComputeJobResult job : results) {
        if (job.getException() != null)
            continue;
        job.<Map<PartitionKeyV2, PartitionHashRecordV2>>getData().forEach((k, v) -> hashes.computeIfAbsent(k, k0 -> new ArrayList<>()).add(v));
    }
    Object mustBeNull = jobResults.putIfAbsent(cls, hashes);
    assertNull(mustBeNull);
}
Also used : ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) PartitionKeyV2(org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2) PartitionHashRecordV2(org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) ArrayList(java.util.ArrayList) ComputeJobResult(org.apache.ignite.compute.ComputeJobResult)

Example 4 with PartitionKeyV2

use of org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2 in project ignite by apache.

the class IgniteClusterSnapshotCheckTest method testClusterSnapshotCheckHashesSameAsIdleVerifyHashes.

/**
 * @throws Exception If fails.
 */
@Test
public void testClusterSnapshotCheckHashesSameAsIdleVerifyHashes() throws Exception {
    Random rnd = new Random();
    CacheConfiguration<Integer, Value> ccfg = txCacheConfig(new CacheConfiguration<>(DEFAULT_CACHE_NAME));
    IgniteEx ignite = startGridsWithCache(1, CACHE_KEYS_RANGE, k -> new Value(new byte[rnd.nextInt(32768)]), ccfg);
    ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get();
    IdleVerifyResultV2 idleVerifyRes = ignite.compute().execute(new TestVisorBackupPartitionsTask(), new VisorIdleVerifyTaskArg(new HashSet<>(singletonList(ccfg.getName())), new HashSet<>(), false, CacheFilterEnum.USER, true));
    IdleVerifyResultV2 snpVerifyRes = ignite.compute().execute(new TestSnapshotPartitionsVerifyTask(), new SnapshotPartitionsVerifyTaskArg(new HashSet<>(), Collections.singletonMap(ignite.cluster().localNode(), Collections.singletonList(snp(ignite).readSnapshotMetadata(SNAPSHOT_NAME, (String) ignite.configuration().getConsistentId()))))).idleVerifyResult();
    Map<PartitionKeyV2, List<PartitionHashRecordV2>> idleVerifyHashes = jobResults.get(TestVisorBackupPartitionsTask.class);
    Map<PartitionKeyV2, List<PartitionHashRecordV2>> snpCheckHashes = jobResults.get(TestVisorBackupPartitionsTask.class);
    assertFalse(F.isEmpty(idleVerifyHashes));
    assertFalse(F.isEmpty(snpCheckHashes));
    assertEquals(idleVerifyHashes, snpCheckHashes);
    assertEquals(idleVerifyRes, snpVerifyRes);
}
Also used : VisorIdleVerifyTaskArg(org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg) Random(java.util.Random) PartitionKeyV2(org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2) IgniteEx(org.apache.ignite.internal.IgniteEx) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) ArrayList(java.util.ArrayList) IdleVerifyResultV2(org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)4 List (java.util.List)4 PartitionKeyV2 (org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2)4 HashMap (java.util.HashMap)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 IdleVerifyResultV2 (org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2)3 PartitionHashRecordV2 (org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2)3 IOException (java.io.IOException)2 Collections.singletonList (java.util.Collections.singletonList)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)2 IgniteException (org.apache.ignite.IgniteException)2 ClusterNode (org.apache.ignite.cluster.ClusterNode)2 GridStringBuilder (org.apache.ignite.internal.util.GridStringBuilder)2 File (java.io.File)1 ByteBuffer (java.nio.ByteBuffer)1 ByteOrder (java.nio.ByteOrder)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1