Examples with CacheGroupContext org.apache.ignite.internal.processors.cache.CacheGroupContext used on opensource projects

Example 96 with CacheGroupContext

use of org.apache.ignite.internal.processors.cache.CacheGroupContext in project ignite by apache.

the class WalRecoveryTxLogicalRecordsTest method rows.

/**
 * @param ignite Ignite.
 * @param part Partition.
 * @param from From counter.
 * @param to To counter.
 */
private List<CacheDataRow> rows(Ignite ignite, int part, long from, long to) throws IgniteCheckedException {
    CacheGroupContext grp = ((IgniteEx) ignite).context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
    IgniteCacheOffheapManager offh = grp.offheap();
    AffinityTopologyVersion topVer = grp.affinity().lastVersion();
    IgniteDhtDemandedPartitionsMap map = new IgniteDhtDemandedPartitionsMap();
    map.addHistorical(part, from, to, PARTS);
    List<CacheDataRow> rows = new ArrayList<>();
    WALPointer ptr = reserveWalPointerForIterator(grp.shared());
    try (IgniteRebalanceIterator it = offh.rebalanceIterator(map, topVer)) {
        assertNotNull(it);
        while (it.hasNextX()) rows.add(it.next());
    } finally {
        releaseWalPointerForIterator(grp.shared(), ptr);
    }
    return rows;
}

Example 97 with CacheGroupContext

use of org.apache.ignite.internal.processors.cache.CacheGroupContext in project ignite by apache.

the class ValidateIndexesClosure method checkSizes.

/**
 * Checking size of records in cache and indexes with a record into
 * {@code checkSizeRes} if they are not equal.
 *
 * @param cacheSizesFutures Futures calculating size of records in caches.
 * @param idxSizeFutures Futures calculating size of indexes of caches.
 * @param checkSizeRes Result of size check.
 */
private void checkSizes(List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizesFutures, List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures, Map<String, ValidateIndexesCheckSizeResult> checkSizeRes) throws ExecutionException, InterruptedException {
    if (!checkSizes)
        return;
    Map<Integer, CacheSize> cacheSizeTotal = new HashMap<>();
    for (T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>> cacheSizeFut : cacheSizesFutures) {
        CacheGroupContext cacheGrpCtx = cacheSizeFut.get1();
        CacheSize cacheSize = cacheSizeFut.get3().get();
        Throwable cacheSizeErr = cacheSize.err;
        int grpId = cacheGrpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId) && nonNull(cacheSizeErr)) {
            checkSizeRes.computeIfAbsent(cacheGrpInfo(cacheGrpCtx), s -> new ValidateIndexesCheckSizeResult(0, new ArrayList<>())).issues().add(new ValidateIndexesCheckSizeIssue(null, 0, cacheSizeErr));
        } else {
            cacheSizeTotal.computeIfAbsent(grpId, i -> new CacheSize(null, new HashMap<>())).merge(cacheSize.cacheSizePerTbl);
        }
    }
    for (T3<GridCacheContext, Index, Future<T2<Throwable, Long>>> idxSizeFut : idxSizeFutures) {
        GridCacheContext cacheCtx = idxSizeFut.get1();
        int grpId = cacheCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            continue;
        Index idx = idxSizeFut.get2();
        String tblName = idx.getTable().getName();
        AtomicLong cacheSizeObj = cacheSizeTotal.get(grpId).cacheSizePerTbl.getOrDefault(cacheCtx.cacheId(), emptyMap()).get(tblName);
        long cacheSizeByTbl = isNull(cacheSizeObj) ? 0L : cacheSizeObj.get();
        T2<Throwable, Long> idxSizeRes = idxSizeFut.get3().get();
        Throwable err = idxSizeRes.get1();
        long idxSize = idxSizeRes.get2();
        if (isNull(err) && idxSize != cacheSizeByTbl)
            err = new IgniteException("Cache and index size not same.");
        if (nonNull(err)) {
            checkSizeRes.computeIfAbsent("[" + cacheGrpInfo(cacheCtx.group()) + ", " + cacheInfo(cacheCtx) + ", tableName=" + tblName + "]", s -> new ValidateIndexesCheckSizeResult(cacheSizeByTbl, new ArrayList<>())).issues().add(new ValidateIndexesCheckSizeIssue(idx.getName(), idxSize, err));
        }
    }
}

Example 98 with CacheGroupContext

use of org.apache.ignite.internal.processors.cache.CacheGroupContext in project ignite by apache.

the class ValidateIndexesClosure method calcCacheSize.

/**
 * Calculation of caches size with divided by tables.
 *
 * @param grpCtx Cache group context.
 * @param locPart Local partition.
 * @return Cache size representation object.
 */
private CacheSize calcCacheSize(CacheGroupContext grpCtx, GridDhtLocalPartition locPart) {
    try {
        if (validateCtx.isCancelled())
            return new CacheSize(null, emptyMap());
        @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
        int grpId = grpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            return new CacheSize(null, null);
        boolean reserve = false;
        int partId = locPart.id();
        try {
            if (!(reserve = locPart.reserve()))
                throw new IgniteException("Can't reserve partition");
            if (locPart.state() != OWNING)
                throw new IgniteException("Partition not in state " + OWNING);
            Map<Integer, Map<String, AtomicLong>> cacheSizeByTbl = new HashMap<>();
            GridIterator<CacheDataRow> partIter = grpCtx.offheap().partitionIterator(partId);
            GridQueryProcessor qryProcessor = ignite.context().query();
            IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
            while (partIter.hasNextX() && !failCalcCacheSizeGrpIds.contains(grpId)) {
                CacheDataRow cacheDataRow = partIter.nextX();
                int cacheId = cacheDataRow.cacheId();
                GridCacheContext cacheCtx = cacheId == 0 ? grpCtx.singleCacheContext() : grpCtx.shared().cacheContext(cacheId);
                if (cacheCtx == null)
                    throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
                if (cacheDataRow.link() == 0L)
                    throw new IgniteException("Contains invalid partition row, possibly deleted");
                String cacheName = cacheCtx.name();
                QueryTypeDescriptorImpl qryTypeDesc = qryProcessor.typeByValue(cacheName, cacheCtx.cacheObjectContext(), cacheDataRow.key(), cacheDataRow.value(), true);
                if (isNull(qryTypeDesc))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                String tableName = qryTypeDesc.tableName();
                GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, tableName);
                if (isNull(gridH2Tbl))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                cacheSizeByTbl.computeIfAbsent(cacheCtx.cacheId(), i -> new HashMap<>()).computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
            }
            PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
            if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
                throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " + "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
            }
            return new CacheSize(null, cacheSizeByTbl);
        } catch (Throwable t) {
            IgniteException cacheSizeErr = new IgniteException("Cache size calculation error [" + cacheGrpInfo(grpCtx) + ", locParId=" + partId + ", err=" + t.getMessage() + "]", t);
            error(log, cacheSizeErr);
            failCalcCacheSizeGrpIds.add(grpId);
            return new CacheSize(cacheSizeErr, null);
        } finally {
            if (reserve)
                locPart.release();
        }
    } finally {
        processedCacheSizePartitions.incrementAndGet();
        printProgressOfIndexValidationIfNeeded();
    }
}

Example 99 with CacheGroupContext

use of org.apache.ignite.internal.processors.cache.CacheGroupContext in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}

Example 100 with CacheGroupContext

use of org.apache.ignite.internal.processors.cache.CacheGroupContext in project ignite by apache.

the class IgniteWalRecoveryTest method testApplyDeltaRecords.

/**
 * @throws Exception if failed.
 */
@Test
public void testApplyDeltaRecords() throws Exception {
    IgniteEx ignite0 = (IgniteEx) startGrid("node0");
    ignite0.cluster().active(true);
    IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
    for (int i = 0; i < 1000; i++) cache0.put(i, new IndexedObject(i));
    GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    db.waitForCheckpoint("test");
    db.enableCheckpoints(false).get();
    // Log something to know where to start.
    WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
    info("Replay marker: " + ptr);
    for (int i = 1000; i < 5000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    for (int i = 2_000; i < 3_000; i++) cache0.remove(i);
    info("Done removes...");
    for (int i = 5000; i < 6000; i++) cache0.put(i, new IndexedObject(i));
    info("Done puts...");
    Map<FullPageId, byte[]> rolledPages = new HashMap<>();
    int pageSize = sharedCtx.database().pageSize();
    ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
    buf.order(ByteOrder.nativeOrder());
    // Now check that deltas can be correctly applied.
    try (WALIterator it = sharedCtx.wal().replay(ptr)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof PageSnapshot) {
                PageSnapshot page = (PageSnapshot) rec;
                CacheGroupContext gctx = sharedCtx.cache().cacheGroup(page.groupId());
                int realPageSize = gctx == null ? pageSize : gctx.dataRegion().pageMemory().realPageSize(page.groupId());
                byte[] pageData = page.pageData();
                if (pageData.length < realPageSize) {
                    buf.clear();
                    buf.put(pageData);
                    buf.flip();
                    sharedCtx.kernalContext().compress().decompressPage(buf, realPageSize);
                    pageData = new byte[realPageSize];
                    buf.position(0);
                    buf.get(pageData);
                }
                rolledPages.put(page.fullPageId(), pageData);
            } else if (rec instanceof PageDeltaRecord) {
                PageDeltaRecord delta = (PageDeltaRecord) rec;
                FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
                byte[] pageData = rolledPages.get(fullId);
                if (pageData == null) {
                    pageData = new byte[pageSize];
                    rolledPages.put(fullId, pageData);
                }
                assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
                buf.clear();
                buf.put(pageData);
                buf.flip();
                delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(), GridUnsafe.bufferAddress(buf));
                buf.get(pageData);
            }
        }
    }
    info("Done apply...");
    PageMemoryEx pageMem = (PageMemoryEx) db.dataRegion(null).pageMemory();
    ByteBuffer bufWal = ByteBuffer.allocateDirect(pageSize);
    for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
        FullPageId fullId = entry.getKey();
        ignite0.context().cache().context().database().checkpointReadLock();
        try {
            long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), IoStatisticsHolderNoOp.INSTANCE, true);
            try {
                long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
                try {
                    byte[] data = entry.getValue();
                    if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
                        // Skip tracking pages.
                        continue;
                    // compacted pages in case of compaction is used.
                    if (walPageCompression != null && PageIO.getPageIO(bufPtr) instanceof CompactablePageIO) {
                        CompactablePageIO pageIO = PageIO.getPageIO(bufPtr);
                        buf.clear();
                        bufWal.clear();
                        int realPageSize = data.length;
                        pageIO.compactPage(GridUnsafe.wrapPointer(bufPtr, realPageSize), buf, realPageSize);
                        pageIO.compactPage(ByteBuffer.wrap(data).order(ByteOrder.nativeOrder()), bufWal, realPageSize);
                        bufPtr = GridUnsafe.bufferAddress(buf);
                        data = new byte[bufWal.limit()];
                        bufWal.rewind();
                        bufWal.get(data);
                    }
                    for (int i = 0; i < data.length; i++) assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
                } finally {
                    pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
                }
            } finally {
                pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
            }
        } finally {
            ignite0.context().cache().context().database().checkpointReadUnlock();
        }
    }
    ignite0.close();
}