Search in sources :

Example 86 with GridDhtLocalPartition

use of org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition in project ignite by apache.

the class ValidateIndexesClosure method checkSizes.

/**
 * Checking size of records in cache and indexes with a record into
 * {@code checkSizeRes} if they are not equal.
 *
 * @param cacheSizesFutures Futures calculating size of records in caches.
 * @param idxSizeFutures Futures calculating size of indexes of caches.
 * @param checkSizeRes Result of size check.
 */
private void checkSizes(List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizesFutures, List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures, Map<String, ValidateIndexesCheckSizeResult> checkSizeRes) throws ExecutionException, InterruptedException {
    if (!checkSizes)
        return;
    Map<Integer, CacheSize> cacheSizeTotal = new HashMap<>();
    for (T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>> cacheSizeFut : cacheSizesFutures) {
        CacheGroupContext cacheGrpCtx = cacheSizeFut.get1();
        CacheSize cacheSize = cacheSizeFut.get3().get();
        Throwable cacheSizeErr = cacheSize.err;
        int grpId = cacheGrpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId) && nonNull(cacheSizeErr)) {
            checkSizeRes.computeIfAbsent(cacheGrpInfo(cacheGrpCtx), s -> new ValidateIndexesCheckSizeResult(0, new ArrayList<>())).issues().add(new ValidateIndexesCheckSizeIssue(null, 0, cacheSizeErr));
        } else {
            cacheSizeTotal.computeIfAbsent(grpId, i -> new CacheSize(null, new HashMap<>())).merge(cacheSize.cacheSizePerTbl);
        }
    }
    for (T3<GridCacheContext, Index, Future<T2<Throwable, Long>>> idxSizeFut : idxSizeFutures) {
        GridCacheContext cacheCtx = idxSizeFut.get1();
        int grpId = cacheCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            continue;
        Index idx = idxSizeFut.get2();
        String tblName = idx.getTable().getName();
        AtomicLong cacheSizeObj = cacheSizeTotal.get(grpId).cacheSizePerTbl.getOrDefault(cacheCtx.cacheId(), emptyMap()).get(tblName);
        long cacheSizeByTbl = isNull(cacheSizeObj) ? 0L : cacheSizeObj.get();
        T2<Throwable, Long> idxSizeRes = idxSizeFut.get3().get();
        Throwable err = idxSizeRes.get1();
        long idxSize = idxSizeRes.get2();
        if (isNull(err) && idxSize != cacheSizeByTbl)
            err = new IgniteException("Cache and index size not same.");
        if (nonNull(err)) {
            checkSizeRes.computeIfAbsent("[" + cacheGrpInfo(cacheCtx.group()) + ", " + cacheInfo(cacheCtx) + ", tableName=" + tblName + "]", s -> new ValidateIndexesCheckSizeResult(cacheSizeByTbl, new ArrayList<>())).issues().add(new ValidateIndexesCheckSizeIssue(idx.getName(), idxSize, err));
        }
    }
}
Also used : Collections.shuffle(java.util.Collections.shuffle) IgniteEx(org.apache.ignite.internal.IgniteEx) DbException(org.h2.message.DbException) H2Utils(org.apache.ignite.internal.processors.query.h2.H2Utils) Index(org.h2.index.Index) IdleVerifyUtility.checkPartitionsPageCrcSum(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum) ConnectionManager(org.apache.ignite.internal.processors.query.h2.ConnectionManager) Future(java.util.concurrent.Future) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) X(org.apache.ignite.internal.util.typedef.X) BPlusTree(org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree) JdbcConnection(org.h2.jdbc.JdbcConnection) Objects.isNull(java.util.Objects.isNull) LoggerResource(org.apache.ignite.resources.LoggerResource) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) IgniteInstanceResource(org.apache.ignite.resources.IgniteInstanceResource) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) Executors(java.util.concurrent.Executors) GRID_NOT_IDLE_MSG(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) Cursor(org.h2.index.Cursor) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) Entry(java.util.Map.Entry) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) Objects.nonNull(java.util.Objects.nonNull) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) CacheObjectUtils(org.apache.ignite.internal.processors.cache.CacheObjectUtils) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) IgniteLogger(org.apache.ignite.IgniteLogger) CorruptedTreeException(org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException) Supplier(java.util.function.Supplier) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) IgniteCallable(org.apache.ignite.lang.IgniteCallable) HashSet(java.util.HashSet) IdleVerifyUtility.formatUpdateCountersDiff(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff) Session(org.h2.engine.Session) Collections.newSetFromMap(java.util.Collections.newSetFromMap) IdleVerifyUtility.compareUpdateCounters(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters) MvccUtils(org.apache.ignite.internal.processors.cache.mvcc.MvccUtils) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IdleVerifyUtility.getUpdateCountersSnapshot(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot) GridH2RowDescriptor(org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ExecutorService(java.util.concurrent.ExecutorService) F(org.apache.ignite.internal.util.typedef.F) Collections.emptyMap(java.util.Collections.emptyMap) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) PartitionUpdateCounter(org.apache.ignite.internal.processors.cache.PartitionUpdateCounter) QueryTypeDescriptorImpl(org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl) T3(org.apache.ignite.internal.util.typedef.T3) GridIterator(org.apache.ignite.internal.util.lang.GridIterator) T2(org.apache.ignite.internal.util.typedef.T2) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) IgniteCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) MvccSnapshot(org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot) H2CacheRow(org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow) Collections(java.util.Collections) MvccQueryTracker(org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker) QueryContext(org.apache.ignite.internal.processors.query.h2.opt.QueryContext) IgniteUtils.error(org.apache.ignite.internal.util.IgniteUtils.error) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Index(org.h2.index.Index) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicLong(java.util.concurrent.atomic.AtomicLong) IgniteException(org.apache.ignite.IgniteException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Future(java.util.concurrent.Future) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext)

Example 87 with GridDhtLocalPartition

use of org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition in project ignite by apache.

the class ValidateIndexesClosure method calcCacheSize.

/**
 * Calculation of caches size with divided by tables.
 *
 * @param grpCtx Cache group context.
 * @param locPart Local partition.
 * @return Cache size representation object.
 */
private CacheSize calcCacheSize(CacheGroupContext grpCtx, GridDhtLocalPartition locPart) {
    try {
        if (validateCtx.isCancelled())
            return new CacheSize(null, emptyMap());
        @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
        int grpId = grpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            return new CacheSize(null, null);
        boolean reserve = false;
        int partId = locPart.id();
        try {
            if (!(reserve = locPart.reserve()))
                throw new IgniteException("Can't reserve partition");
            if (locPart.state() != OWNING)
                throw new IgniteException("Partition not in state " + OWNING);
            Map<Integer, Map<String, AtomicLong>> cacheSizeByTbl = new HashMap<>();
            GridIterator<CacheDataRow> partIter = grpCtx.offheap().partitionIterator(partId);
            GridQueryProcessor qryProcessor = ignite.context().query();
            IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
            while (partIter.hasNextX() && !failCalcCacheSizeGrpIds.contains(grpId)) {
                CacheDataRow cacheDataRow = partIter.nextX();
                int cacheId = cacheDataRow.cacheId();
                GridCacheContext cacheCtx = cacheId == 0 ? grpCtx.singleCacheContext() : grpCtx.shared().cacheContext(cacheId);
                if (cacheCtx == null)
                    throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
                if (cacheDataRow.link() == 0L)
                    throw new IgniteException("Contains invalid partition row, possibly deleted");
                String cacheName = cacheCtx.name();
                QueryTypeDescriptorImpl qryTypeDesc = qryProcessor.typeByValue(cacheName, cacheCtx.cacheObjectContext(), cacheDataRow.key(), cacheDataRow.value(), true);
                if (isNull(qryTypeDesc))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                String tableName = qryTypeDesc.tableName();
                GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, tableName);
                if (isNull(gridH2Tbl))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                cacheSizeByTbl.computeIfAbsent(cacheCtx.cacheId(), i -> new HashMap<>()).computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
            }
            PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
            if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
                throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " + "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
            }
            return new CacheSize(null, cacheSizeByTbl);
        } catch (Throwable t) {
            IgniteException cacheSizeErr = new IgniteException("Cache size calculation error [" + cacheGrpInfo(grpCtx) + ", locParId=" + partId + ", err=" + t.getMessage() + "]", t);
            error(log, cacheSizeErr);
            failCalcCacheSizeGrpIds.add(grpId);
            return new CacheSize(cacheSizeErr, null);
        } finally {
            if (reserve)
                locPart.release();
        }
    } finally {
        processedCacheSizePartitions.incrementAndGet();
        printProgressOfIndexValidationIfNeeded();
    }
}
Also used : CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) Collections.shuffle(java.util.Collections.shuffle) IgniteEx(org.apache.ignite.internal.IgniteEx) DbException(org.h2.message.DbException) H2Utils(org.apache.ignite.internal.processors.query.h2.H2Utils) Index(org.h2.index.Index) IdleVerifyUtility.checkPartitionsPageCrcSum(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum) ConnectionManager(org.apache.ignite.internal.processors.query.h2.ConnectionManager) Future(java.util.concurrent.Future) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) X(org.apache.ignite.internal.util.typedef.X) BPlusTree(org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree) JdbcConnection(org.h2.jdbc.JdbcConnection) Objects.isNull(java.util.Objects.isNull) LoggerResource(org.apache.ignite.resources.LoggerResource) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) IgniteInstanceResource(org.apache.ignite.resources.IgniteInstanceResource) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) Executors(java.util.concurrent.Executors) GRID_NOT_IDLE_MSG(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) Cursor(org.h2.index.Cursor) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) Entry(java.util.Map.Entry) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) Objects.nonNull(java.util.Objects.nonNull) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) CacheObjectUtils(org.apache.ignite.internal.processors.cache.CacheObjectUtils) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) IgniteLogger(org.apache.ignite.IgniteLogger) CorruptedTreeException(org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException) Supplier(java.util.function.Supplier) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) IgniteCallable(org.apache.ignite.lang.IgniteCallable) HashSet(java.util.HashSet) IdleVerifyUtility.formatUpdateCountersDiff(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff) Session(org.h2.engine.Session) Collections.newSetFromMap(java.util.Collections.newSetFromMap) IdleVerifyUtility.compareUpdateCounters(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters) MvccUtils(org.apache.ignite.internal.processors.cache.mvcc.MvccUtils) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IdleVerifyUtility.getUpdateCountersSnapshot(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot) GridH2RowDescriptor(org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ExecutorService(java.util.concurrent.ExecutorService) F(org.apache.ignite.internal.util.typedef.F) Collections.emptyMap(java.util.Collections.emptyMap) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) PartitionUpdateCounter(org.apache.ignite.internal.processors.cache.PartitionUpdateCounter) QueryTypeDescriptorImpl(org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl) T3(org.apache.ignite.internal.util.typedef.T3) GridIterator(org.apache.ignite.internal.util.lang.GridIterator) T2(org.apache.ignite.internal.util.typedef.T2) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) IgniteCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) MvccSnapshot(org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot) H2CacheRow(org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow) Collections(java.util.Collections) MvccQueryTracker(org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker) QueryContext(org.apache.ignite.internal.processors.query.h2.opt.QueryContext) IgniteUtils.error(org.apache.ignite.internal.util.IgniteUtils.error) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) QueryTypeDescriptorImpl(org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) PartitionUpdateCounter(org.apache.ignite.internal.processors.cache.PartitionUpdateCounter) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) AtomicLong(java.util.concurrent.atomic.AtomicLong) IgniteException(org.apache.ignite.IgniteException) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) Collections.newSetFromMap(java.util.Collections.newSetFromMap) Collections.emptyMap(java.util.Collections.emptyMap) Nullable(org.jetbrains.annotations.Nullable)

Example 88 with GridDhtLocalPartition

use of org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}
Also used : Collections.shuffle(java.util.Collections.shuffle) IgniteEx(org.apache.ignite.internal.IgniteEx) DbException(org.h2.message.DbException) H2Utils(org.apache.ignite.internal.processors.query.h2.H2Utils) Index(org.h2.index.Index) IdleVerifyUtility.checkPartitionsPageCrcSum(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum) ConnectionManager(org.apache.ignite.internal.processors.query.h2.ConnectionManager) Future(java.util.concurrent.Future) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) X(org.apache.ignite.internal.util.typedef.X) BPlusTree(org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree) JdbcConnection(org.h2.jdbc.JdbcConnection) Objects.isNull(java.util.Objects.isNull) LoggerResource(org.apache.ignite.resources.LoggerResource) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) IgniteInstanceResource(org.apache.ignite.resources.IgniteInstanceResource) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) Executors(java.util.concurrent.Executors) GRID_NOT_IDLE_MSG(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) Cursor(org.h2.index.Cursor) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) Entry(java.util.Map.Entry) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) Objects.nonNull(java.util.Objects.nonNull) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) CacheObjectUtils(org.apache.ignite.internal.processors.cache.CacheObjectUtils) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) IgniteLogger(org.apache.ignite.IgniteLogger) CorruptedTreeException(org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException) Supplier(java.util.function.Supplier) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) IgniteCallable(org.apache.ignite.lang.IgniteCallable) HashSet(java.util.HashSet) IdleVerifyUtility.formatUpdateCountersDiff(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff) Session(org.h2.engine.Session) Collections.newSetFromMap(java.util.Collections.newSetFromMap) IdleVerifyUtility.compareUpdateCounters(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters) MvccUtils(org.apache.ignite.internal.processors.cache.mvcc.MvccUtils) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IdleVerifyUtility.getUpdateCountersSnapshot(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot) GridH2RowDescriptor(org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ExecutorService(java.util.concurrent.ExecutorService) F(org.apache.ignite.internal.util.typedef.F) Collections.emptyMap(java.util.Collections.emptyMap) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) PartitionUpdateCounter(org.apache.ignite.internal.processors.cache.PartitionUpdateCounter) QueryTypeDescriptorImpl(org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl) T3(org.apache.ignite.internal.util.typedef.T3) GridIterator(org.apache.ignite.internal.util.lang.GridIterator) T2(org.apache.ignite.internal.util.typedef.T2) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) IgniteCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) MvccSnapshot(org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot) H2CacheRow(org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow) Collections(java.util.Collections) MvccQueryTracker(org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker) QueryContext(org.apache.ignite.internal.processors.query.h2.opt.QueryContext) IgniteUtils.error(org.apache.ignite.internal.util.IgniteUtils.error) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Index(org.h2.index.Index) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteException(org.apache.ignite.IgniteException) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) Collections.newSetFromMap(java.util.Collections.newSetFromMap) Collections.emptyMap(java.util.Collections.emptyMap) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) ExecutionException(java.util.concurrent.ExecutionException) T2(org.apache.ignite.internal.util.typedef.T2) T3(org.apache.ignite.internal.util.typedef.T3) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Future(java.util.concurrent.Future)

Example 89 with GridDhtLocalPartition

use of org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition in project ignite by apache.

the class IgniteCacheLockPartitionOnAffinityRunTest method getOrganizationCountFromPartitionMap.

/**
 * @param ignite Ignite.
 * @param orgId Organization id.
 * @return Count of found Person object with specified orgId
 * @throws Exception If failed.
 */
private static int getOrganizationCountFromPartitionMap(final IgniteEx ignite, int orgId) throws Exception {
    int part = ignite.affinity(Organization.class.getSimpleName()).partition(orgId);
    GridCacheAdapter<?, ?> cacheAdapterOrg = ignite.context().cache().internalCache(Organization.class.getSimpleName());
    GridDhtLocalPartition pOrgs = cacheAdapterOrg.context().topology().localPartition(part, AffinityTopologyVersion.NONE, false);
    int cnt = 0;
    GridCursor<? extends CacheDataRow> c = pOrgs.dataStore().cursor();
    CacheObjectContext ctx = cacheAdapterOrg.context().cacheObjectContext();
    while (c.next()) {
        CacheDataRow e = c.get();
        Integer k = e.key().value(ctx, false);
        Organization org = e.value().value(ctx, false);
        if (org != null && org.getId() == orgId)
            ++cnt;
    }
    return cnt;
}
Also used : CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition)

Example 90 with GridDhtLocalPartition

use of org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition in project ignite by apache.

the class IgniteCacheLockPartitionOnAffinityRunTest method primaryPartition.

/**
 * @param ignite Ignite instance.
 * @param orgId Organization ID.
 * @return {@code true} if partition for the given organization ID is primary on the given node.
 */
private static boolean primaryPartition(IgniteEx ignite, int orgId) {
    int part = ignite.affinity(Organization.class.getSimpleName()).partition(orgId);
    GridCacheAdapter<?, ?> cacheAdapterPers = ignite.context().cache().internalCache(Person.class.getSimpleName());
    GridDhtLocalPartition pPers = cacheAdapterPers.context().topology().localPartition(part, AffinityTopologyVersion.NONE, false);
    return pPers.primary(AffinityTopologyVersion.NONE);
}
Also used : GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition)

Aggregations

GridDhtLocalPartition (org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition)95 GridDhtPartitionTopology (org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology)21 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)19 IgniteEx (org.apache.ignite.internal.IgniteEx)19 CacheGroupContext (org.apache.ignite.internal.processors.cache.CacheGroupContext)19 ArrayList (java.util.ArrayList)18 Map (java.util.Map)18 Test (org.junit.Test)18 AffinityTopologyVersion (org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion)16 GridCommonAbstractTest (org.apache.ignite.testframework.junits.common.GridCommonAbstractTest)16 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)15 ClusterNode (org.apache.ignite.cluster.ClusterNode)15 GridCacheContext (org.apache.ignite.internal.processors.cache.GridCacheContext)15 HashMap (java.util.HashMap)14 HashSet (java.util.HashSet)13 AtomicLong (java.util.concurrent.atomic.AtomicLong)13 CacheDataRow (org.apache.ignite.internal.processors.cache.persistence.CacheDataRow)13 Ignite (org.apache.ignite.Ignite)12 KeyCacheObject (org.apache.ignite.internal.processors.cache.KeyCacheObject)12 IgniteException (org.apache.ignite.IgniteException)11