Search in sources :

Example 6 with H2TreeIndexBase

use of org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}
Also used : Collections.shuffle(java.util.Collections.shuffle) IgniteEx(org.apache.ignite.internal.IgniteEx) DbException(org.h2.message.DbException) H2Utils(org.apache.ignite.internal.processors.query.h2.H2Utils) Index(org.h2.index.Index) IdleVerifyUtility.checkPartitionsPageCrcSum(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum) ConnectionManager(org.apache.ignite.internal.processors.query.h2.ConnectionManager) Future(java.util.concurrent.Future) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) X(org.apache.ignite.internal.util.typedef.X) BPlusTree(org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree) JdbcConnection(org.h2.jdbc.JdbcConnection) Objects.isNull(java.util.Objects.isNull) LoggerResource(org.apache.ignite.resources.LoggerResource) FilePageStore(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) IgniteInstanceResource(org.apache.ignite.resources.IgniteInstanceResource) FilePageStoreManager(org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager) OWNING(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING) CacheDataRow(org.apache.ignite.internal.processors.cache.persistence.CacheDataRow) Executors(java.util.concurrent.Executors) GRID_NOT_IDLE_MSG(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.GRID_NOT_IDLE_MSG) INDEX_PARTITION(org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION) Cursor(org.h2.index.Cursor) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) Entry(java.util.Map.Entry) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) KeyCacheObject(org.apache.ignite.internal.processors.cache.KeyCacheObject) Objects.nonNull(java.util.Objects.nonNull) FLAG_IDX(org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX) CacheObjectUtils(org.apache.ignite.internal.processors.cache.CacheObjectUtils) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) U(org.apache.ignite.internal.util.typedef.internal.U) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) IgniteLogger(org.apache.ignite.IgniteLogger) CorruptedTreeException(org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException) Supplier(java.util.function.Supplier) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) ArrayList(java.util.ArrayList) GridKernalContext(org.apache.ignite.internal.GridKernalContext) IgniteCallable(org.apache.ignite.lang.IgniteCallable) HashSet(java.util.HashSet) IdleVerifyUtility.formatUpdateCountersDiff(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.formatUpdateCountersDiff) Session(org.h2.engine.Session) Collections.newSetFromMap(java.util.Collections.newSetFromMap) IdleVerifyUtility.compareUpdateCounters(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.compareUpdateCounters) MvccUtils(org.apache.ignite.internal.processors.cache.mvcc.MvccUtils) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) IdleVerifyUtility.getUpdateCountersSnapshot(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.getUpdateCountersSnapshot) GridH2RowDescriptor(org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ExecutorService(java.util.concurrent.ExecutorService) F(org.apache.ignite.internal.util.typedef.F) Collections.emptyMap(java.util.Collections.emptyMap) DynamicCacheDescriptor(org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) PartitionUpdateCounter(org.apache.ignite.internal.processors.cache.PartitionUpdateCounter) QueryTypeDescriptorImpl(org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl) T3(org.apache.ignite.internal.util.typedef.T3) GridIterator(org.apache.ignite.internal.util.lang.GridIterator) T2(org.apache.ignite.internal.util.typedef.T2) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) IgniteCacheDatabaseSharedManager(org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager) GridCacheSharedContext(org.apache.ignite.internal.processors.cache.GridCacheSharedContext) MvccSnapshot(org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot) H2CacheRow(org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow) Collections(java.util.Collections) MvccQueryTracker(org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker) QueryContext(org.apache.ignite.internal.processors.query.h2.opt.QueryContext) IgniteUtils.error(org.apache.ignite.internal.util.IgniteUtils.error) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) IdleVerifyUtility(org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Index(org.h2.index.Index) GridQueryTypeDescriptor(org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor) IgniteException(org.apache.ignite.IgniteException) GridCacheContext(org.apache.ignite.internal.processors.cache.GridCacheContext) GridNotIdleException(org.apache.ignite.internal.processors.cache.verify.GridNotIdleException) GridQueryProcessor(org.apache.ignite.internal.processors.query.GridQueryProcessor) PartitionKey(org.apache.ignite.internal.processors.cache.verify.PartitionKey) IgniteH2Indexing(org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing) CacheGroupContext(org.apache.ignite.internal.processors.cache.CacheGroupContext) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) Collections.newSetFromMap(java.util.Collections.newSetFromMap) Collections.emptyMap(java.util.Collections.emptyMap) IgniteInterruptedException(org.apache.ignite.IgniteInterruptedException) GridH2Table(org.apache.ignite.internal.processors.query.h2.opt.GridH2Table) GridDhtLocalPartition(org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition) ExecutionException(java.util.concurrent.ExecutionException) T2(org.apache.ignite.internal.util.typedef.T2) T3(org.apache.ignite.internal.util.typedef.T3) H2TreeIndexBase(org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Future(java.util.concurrent.Future)

Aggregations

ArrayList (java.util.ArrayList)5 H2TreeIndexBase (org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase)5 Index (org.h2.index.Index)5 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)3 IgniteException (org.apache.ignite.IgniteException)3 GridQueryProcessor (org.apache.ignite.internal.processors.query.GridQueryProcessor)3 IgniteH2Indexing (org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing)3 GridH2Table (org.apache.ignite.internal.processors.query.h2.opt.GridH2Table)3 IndexColumn (org.h2.table.IndexColumn)3 Collection (java.util.Collection)2 IgniteEx (org.apache.ignite.internal.IgniteEx)2 CacheGroupContext (org.apache.ignite.internal.processors.cache.CacheGroupContext)2 GridCacheContext (org.apache.ignite.internal.processors.cache.GridCacheContext)2 GridQueryTypeDescriptor (org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor)2 SpatialIndex (org.h2.index.SpatialIndex)2 Collections (java.util.Collections)1 Collections.emptyMap (java.util.Collections.emptyMap)1 Collections.newSetFromMap (java.util.Collections.newSetFromMap)1 Collections.shuffle (java.util.Collections.shuffle)1 HashMap (java.util.HashMap)1