Examples with PartitionUpdateCounter org.apache.ignite.internal.processors.cache.PartitionUpdateCounter used on opensource projects

Example 26 with PartitionUpdateCounter

use of org.apache.ignite.internal.processors.cache.PartitionUpdateCounter in project ignite by apache.

the class PartitionUpdateCounterTest method testMaxGaps.

/**
 * Test logic for handling gaps limit.
 */
@Test
public void testMaxGaps() {
    PartitionUpdateCounter pc = new PartitionUpdateCounterTrackingImpl(null);
    int i;
    for (i = 1; i <= PartitionUpdateCounterTrackingImpl.MAX_MISSED_UPDATES; i++) pc.update(i * 3, i * 3 + 1);
    i++;
    try {
        pc.update(i * 3, i * 3 + 1);
        fail();
    } catch (Exception e) {
    // Expected.
    }
}

Example 27 with PartitionUpdateCounter

use of org.apache.ignite.internal.processors.cache.PartitionUpdateCounter in project ignite by apache.

the class ValidateIndexesClosure method calcCacheSize.

/**
 * Calculation of caches size with divided by tables.
 *
 * @param grpCtx Cache group context.
 * @param locPart Local partition.
 * @return Cache size representation object.
 */
private CacheSize calcCacheSize(CacheGroupContext grpCtx, GridDhtLocalPartition locPart) {
    try {
        if (validateCtx.isCancelled())
            return new CacheSize(null, emptyMap());
        @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
        int grpId = grpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            return new CacheSize(null, null);
        boolean reserve = false;
        int partId = locPart.id();
        try {
            if (!(reserve = locPart.reserve()))
                throw new IgniteException("Can't reserve partition");
            if (locPart.state() != OWNING)
                throw new IgniteException("Partition not in state " + OWNING);
            Map<Integer, Map<String, AtomicLong>> cacheSizeByTbl = new HashMap<>();
            GridIterator<CacheDataRow> partIter = grpCtx.offheap().partitionIterator(partId);
            GridQueryProcessor qryProcessor = ignite.context().query();
            IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
            while (partIter.hasNextX() && !failCalcCacheSizeGrpIds.contains(grpId)) {
                CacheDataRow cacheDataRow = partIter.nextX();
                int cacheId = cacheDataRow.cacheId();
                GridCacheContext cacheCtx = cacheId == 0 ? grpCtx.singleCacheContext() : grpCtx.shared().cacheContext(cacheId);
                if (cacheCtx == null)
                    throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
                if (cacheDataRow.link() == 0L)
                    throw new IgniteException("Contains invalid partition row, possibly deleted");
                String cacheName = cacheCtx.name();
                QueryTypeDescriptorImpl qryTypeDesc = qryProcessor.typeByValue(cacheName, cacheCtx.cacheObjectContext(), cacheDataRow.key(), cacheDataRow.value(), true);
                if (isNull(qryTypeDesc))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                String tableName = qryTypeDesc.tableName();
                GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, tableName);
                if (isNull(gridH2Tbl))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                cacheSizeByTbl.computeIfAbsent(cacheCtx.cacheId(), i -> new HashMap<>()).computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
            }
            PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
            if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
                throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " + "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
            }
            return new CacheSize(null, cacheSizeByTbl);
        } catch (Throwable t) {
            IgniteException cacheSizeErr = new IgniteException("Cache size calculation error [" + cacheGrpInfo(grpCtx) + ", locParId=" + partId + ", err=" + t.getMessage() + "]", t);
            error(log, cacheSizeErr);
            failCalcCacheSizeGrpIds.add(grpId);
            return new CacheSize(cacheSizeErr, null);
        } finally {
            if (reserve)
                locPart.release();
        }
    } finally {
        processedCacheSizePartitions.incrementAndGet();
        printProgressOfIndexValidationIfNeeded();
    }
}

Example 28 with PartitionUpdateCounter

use of org.apache.ignite.internal.processors.cache.PartitionUpdateCounter in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}

Example 29 with PartitionUpdateCounter

use of org.apache.ignite.internal.processors.cache.PartitionUpdateCounter in project ignite by apache.

the class ValidateIndexesClosure method processPartition.

/**
 * @param grpCtx Group context.
 * @param part Local partition.
 */
private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(CacheGroupContext grpCtx, GridDhtLocalPartition part) {
    if (validateCtx.isCancelled() || !part.reserve())
        return emptyMap();
    ValidateIndexesPartitionResult partRes;
    try {
        if (part.state() != OWNING)
            return emptyMap();
        @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
        partRes = new ValidateIndexesPartitionResult();
        boolean hasMvcc = grpCtx.caches().stream().anyMatch(GridCacheContext::mvccEnabled);
        if (hasMvcc) {
            for (GridCacheContext<?, ?> context : grpCtx.caches()) {
                try (Session session = mvccSession(context)) {
                    MvccSnapshot mvccSnapshot = null;
                    boolean mvccEnabled = context.mvccEnabled();
                    if (mvccEnabled)
                        mvccSnapshot = ((QueryContext) session.getVariable(H2Utils.QCTX_VARIABLE_NAME).getObject()).mvccSnapshot();
                    GridIterator<CacheDataRow> iterator = grpCtx.offheap().cachePartitionIterator(context.cacheId(), part.id(), mvccSnapshot, null);
                    processPartIterator(grpCtx, partRes, session, iterator);
                }
            }
        } else
            processPartIterator(grpCtx, partRes, null, grpCtx.offheap().partitionIterator(part.id()));
        PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
        if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
            throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during index validation " + "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
        }
    } catch (IgniteCheckedException e) {
        error(log, "Failed to process partition [grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "]", e);
        return emptyMap();
    } finally {
        part.release();
        printProgressOfIndexValidationIfNeeded();
    }
    PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
    processedPartitions.incrementAndGet();
    return Collections.singletonMap(partKey, partRes);
}