Examples with PartitionKey org.apache.ignite.internal.processors.cache.verify.PartitionKey used on opensource projects

Example 1 with PartitionKey

use of org.apache.ignite.internal.processors.cache.verify.PartitionKey in project ignite by apache.

the class IdleVerify method legacyCacheIdleVerify.

/**
 * @param client Client.
 * @param clientCfg Client configuration.
 */
private void legacyCacheIdleVerify(GridClient client, GridClientConfiguration clientCfg, Logger logger) throws GridClientException {
    VisorIdleVerifyTaskResult res = executeTask(client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(args.caches(), args.excludeCaches(), args.isSkipZeros(), args.getCacheFilterEnum(), args.idleCheckCrc()), clientCfg);
    Map<PartitionKey, List<PartitionHashRecord>> conflicts = res.getConflicts();
    if (conflicts.isEmpty()) {
        logger.info("The check procedure has finished, no conflicts have been found.");
        logger.info("");
    } else {
        logger.info("The check procedure has finished, found " + conflicts.size() + " conflict partitions.");
        logger.info("");
        for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
            logger.info("Conflict partition: " + entry.getKey());
            logger.info("Partition instances: " + entry.getValue());
        }
    }
}

Example 2 with PartitionKey

use of org.apache.ignite.internal.processors.cache.verify.PartitionKey in project ignite by apache.

the class CacheValidateIndexes method execute.

/**
 * {@inheritDoc}
 */
@Override
public Object execute(GridClientConfiguration clientCfg, Logger logger) throws Exception {
    VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(args.caches(), args.nodeId() != null ? Collections.singleton(args.nodeId()) : null, args.checkFirst(), args.checkThrough(), args.checkCrc(), args.checkSizes());
    try (GridClient client = Command.startClient(clientCfg)) {
        VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(client, "org.apache.ignite.internal.visor.verify.VisorValidateIndexesTask", taskArg, null, clientCfg);
        boolean errors = CommandLogger.printErrors(taskRes.exceptions(), "Index validation failed on nodes:", logger);
        for (Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
            VisorValidateIndexesJobResult jobRes = nodeEntry.getValue();
            if (!jobRes.hasIssues())
                continue;
            errors = true;
            logger.info("Index issues found on node " + nodeEntry.getKey() + ":");
            for (IndexIntegrityCheckIssue is : jobRes.integrityCheckFailures()) logger.info(INDENT + is);
            for (Entry<PartitionKey, ValidateIndexesPartitionResult> e : jobRes.partitionResult().entrySet()) {
                ValidateIndexesPartitionResult res = e.getValue();
                if (!res.issues().isEmpty()) {
                    logger.info(INDENT + join(" ", e.getKey(), e.getValue()));
                    for (IndexValidationIssue is : res.issues()) logger.info(DOUBLE_INDENT + is);
                }
            }
            for (Entry<String, ValidateIndexesPartitionResult> e : jobRes.indexResult().entrySet()) {
                ValidateIndexesPartitionResult res = e.getValue();
                if (!res.issues().isEmpty()) {
                    logger.info(INDENT + join(" ", "SQL Index", e.getKey(), e.getValue()));
                    for (IndexValidationIssue is : res.issues()) logger.info(DOUBLE_INDENT + is);
                }
            }
            for (Entry<String, ValidateIndexesCheckSizeResult> e : jobRes.checkSizeResult().entrySet()) {
                ValidateIndexesCheckSizeResult res = e.getValue();
                Collection<ValidateIndexesCheckSizeIssue> issues = res.issues();
                if (issues.isEmpty())
                    continue;
                logger.info(INDENT + join(" ", "Size check", e.getKey(), res));
                for (ValidateIndexesCheckSizeIssue issue : issues) logger.info(DOUBLE_INDENT + issue);
            }
        }
        if (!errors)
            logger.info("no issues found.");
        else
            logger.severe("issues found (listed above).");
        logger.info("");
        return taskRes;
    }
}

Example 3 with PartitionKey

use of org.apache.ignite.internal.processors.cache.verify.PartitionKey in project ignite by apache.

the class VisorIdleAnalyzeTaskArg method readExternalData.

/**
 * {@inheritDoc}
 */
@Override
protected void readExternalData(byte protoVer, ObjectInput in) throws IOException {
    int grpId = in.readInt();
    int partId = in.readInt();
    String grpName = U.readString(in);
    partKey = new PartitionKey(grpId, partId, grpName);
}

Example 4 with PartitionKey

use of org.apache.ignite.internal.processors.cache.verify.PartitionKey in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}

Example 5 with PartitionKey

use of org.apache.ignite.internal.processors.cache.verify.PartitionKey in project ignite by apache.

the class ValidateIndexesClosure method processPartition.

/**
 * @param grpCtx Group context.
 * @param part Local partition.
 */
private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(CacheGroupContext grpCtx, GridDhtLocalPartition part) {
    if (validateCtx.isCancelled() || !part.reserve())
        return emptyMap();
    ValidateIndexesPartitionResult partRes;
    try {
        if (part.state() != OWNING)
            return emptyMap();
        @Nullable PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy();
        partRes = new ValidateIndexesPartitionResult();
        boolean hasMvcc = grpCtx.caches().stream().anyMatch(GridCacheContext::mvccEnabled);
        if (hasMvcc) {
            for (GridCacheContext<?, ?> context : grpCtx.caches()) {
                try (Session session = mvccSession(context)) {
                    MvccSnapshot mvccSnapshot = null;
                    boolean mvccEnabled = context.mvccEnabled();
                    if (mvccEnabled)
                        mvccSnapshot = ((QueryContext) session.getVariable(H2Utils.QCTX_VARIABLE_NAME).getObject()).mvccSnapshot();
                    GridIterator<CacheDataRow> iterator = grpCtx.offheap().cachePartitionIterator(context.cacheId(), part.id(), mvccSnapshot, null);
                    processPartIterator(grpCtx, partRes, session, iterator);
                }
            }
        } else
            processPartIterator(grpCtx, partRes, null, grpCtx.offheap().partitionIterator(part.id()));
        PartitionUpdateCounter updateCntrAfter = part.dataStore().partUpdateCounter();
        if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
            throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "] changed during index validation " + "[before=" + updateCntrBefore + ", after=" + updateCntrAfter + "]");
        }
    } catch (IgniteCheckedException e) {
        error(log, "Failed to process partition [grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "]", e);
        return emptyMap();
    } finally {
        part.release();
        printProgressOfIndexValidationIfNeeded();
    }
    PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
    processedPartitions.incrementAndGet();
    return Collections.singletonMap(partKey, partRes);
}