Examples with GridIterator org.apache.ignite.internal.util.lang.GridIterator used on opensource projects

Example 1 with GridIterator

use of org.apache.ignite.internal.util.lang.GridIterator in project ignite by apache.

the class IgnitePdsPartitionPreloadTest method preloadPartition.

/**
 * @param execNodeFactory Test node factory.
 * @param preloadMode Preload mode.
 */
private void preloadPartition(Supplier<Ignite> execNodeFactory, PreloadMode preloadMode) throws Exception {
    Ignite crd = startGridsMultiThreaded(GRIDS_CNT);
    Ignite testNode = grid(1);
    Object consistentId = testNode.cluster().localNode().consistentId();
    assertEquals(PRIMARY_NODE, testNode.cluster().localNode().consistentId());
    boolean locCacheMode = testNode.cache(DEFAULT_CACHE_NAME).getConfiguration(CacheConfiguration.class).getCacheMode() == LOCAL;
    Integer key = primaryKey(testNode.cache(DEFAULT_CACHE_NAME));
    int preloadPart = crd.affinity(DEFAULT_CACHE_NAME).partition(key);
    int cnt = 0;
    try (IgniteDataStreamer<Integer, Integer> streamer = testNode.dataStreamer(DEFAULT_CACHE_NAME)) {
        int k = 0;
        while (cnt < ENTRY_CNT) {
            if (testNode.affinity(DEFAULT_CACHE_NAME).partition(k) == preloadPart) {
                streamer.addData(k, k);
                cnt++;
            }
            k++;
        }
    }
    forceCheckpoint();
    stopAllGrids();
    startGridsMultiThreaded(GRIDS_CNT);
    testNode = G.allGrids().stream().filter(ignite -> PRIMARY_NODE.equals(ignite.cluster().localNode().consistentId())).findFirst().get();
    if (!locCacheMode)
        assertEquals(testNode, primaryNode(key, DEFAULT_CACHE_NAME));
    Ignite execNode = execNodeFactory.get();
    switch(preloadMode) {
        case SYNC:
            execNode.cache(DEFAULT_CACHE_NAME).preloadPartition(preloadPart);
            if (locCacheMode) {
                testNode = G.allGrids().stream().filter(ignite -> ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get();
            }
            break;
        case ASYNC:
            execNode.cache(DEFAULT_CACHE_NAME).preloadPartitionAsync(preloadPart).get();
            if (locCacheMode) {
                testNode = G.allGrids().stream().filter(ignite -> ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get();
            }
            break;
        case LOCAL:
            assertTrue(execNode.cache(DEFAULT_CACHE_NAME).localPreloadPartition(preloadPart));
            // For local preloading testNode == execNode
            testNode = execNode;
            break;
    }
    long c0 = testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead();
    // After partition preloading no pages should be read from store.
    GridIterator<CacheDataRow> cursor = ((IgniteEx) testNode).cachex(DEFAULT_CACHE_NAME).context().offheap().cachePartitionIterator(CU.UNDEFINED_CACHE_ID, preloadPart, null, false);
    int realSize = 0;
    while (cursor.hasNext()) {
        realSize++;
        cursor.next();
    }
    assertEquals("Partition has missed some entries", ENTRY_CNT, realSize);
    assertEquals("Read pages count must be same", c0, testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead());
}

Example 2 with GridIterator

use of org.apache.ignite.internal.util.lang.GridIterator in project ignite by apache.

the class VisorFindAndDeleteGarbageInPersistenceClosure method processPartition.

/**
 * @param grpCtx Group context.
 * @param part Local partition.
 */
private Map<Integer, Map<Integer, Long>> processPartition(CacheGroupContext grpCtx, GridDhtLocalPartition part) {
    if (!part.reserve())
        return Collections.emptyMap();
    Map<Integer, Map<Integer, Long>> stoppedCachesForGrpId = new HashMap<>();
    try {
        if (part.state() != GridDhtPartitionState.OWNING)
            return Collections.emptyMap();
        GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
        while (it.hasNextX()) {
            CacheDataRow row = it.nextX();
            if (row.cacheId() == 0)
                break;
            int cacheId = row.cacheId();
            GridCacheContext cacheCtx = grpCtx.shared().cacheContext(row.cacheId());
            if (cacheCtx == null)
                stoppedCachesForGrpId.computeIfAbsent(grpCtx.groupId(), (x) -> new HashMap<>()).compute(cacheId, (x, y) -> y == null ? 1 : y + 1);
        }
    } catch (IgniteCheckedException e) {
        U.error(log, "Failed to process partition [grpId=" + grpCtx.groupId() + ", partId=" + part.id() + "]", e);
        return Collections.emptyMap();
    } finally {
        part.release();
    }
    processedPartitions.incrementAndGet();
    printProgressIfNeeded();
    return stoppedCachesForGrpId;
}

Example 3 with GridIterator

use of org.apache.ignite.internal.util.lang.GridIterator in project ignite by apache.

the class ValidateIndexesClosure method calcCacheSize.

/**
 * Calculation of caches size with divided by tables.
 *
 * @param grpCtx Cache group context.
 * @param locPart Local partition.
 * @return Cache size representation object.
 */
private CacheSize calcCacheSize(CacheGroupContext grpCtx, GridDhtLocalPartition locPart) {
    try {
        if (validateCtx.isCancelled())
            return new CacheSize(null, emptyMap());
        @Nullable PartitionUpdateCounter updCntr = locPart.dataStore().partUpdateCounter();
        PartitionUpdateCounter updateCntrBefore = updCntr == null ? updCntr : updCntr.copy();
        int grpId = grpCtx.groupId();
        if (failCalcCacheSizeGrpIds.contains(grpId))
            return new CacheSize(null, null);
        boolean reserve = false;
        int partId = locPart.id();
        try {
            if (!(reserve = locPart.reserve()))
                throw new IgniteException("Can't reserve partition");
            if (locPart.state() != OWNING)
                throw new IgniteException("Partition not in state " + OWNING);
            Map<Integer, Map<String, AtomicLong>> cacheSizeByTbl = new HashMap<>();
            GridIterator<CacheDataRow> partIter = grpCtx.offheap().partitionIterator(partId);
            GridQueryProcessor qryProcessor = ignite.context().query();
            IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
            while (partIter.hasNextX() && !failCalcCacheSizeGrpIds.contains(grpId)) {
                CacheDataRow cacheDataRow = partIter.nextX();
                int cacheId = cacheDataRow.cacheId();
                GridCacheContext cacheCtx = cacheId == 0 ? grpCtx.singleCacheContext() : grpCtx.shared().cacheContext(cacheId);
                if (cacheCtx == null)
                    throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
                if (cacheDataRow.link() == 0L)
                    throw new IgniteException("Contains invalid partition row, possibly deleted");
                String cacheName = cacheCtx.name();
                QueryTypeDescriptorImpl qryTypeDesc = qryProcessor.typeByValue(cacheName, cacheCtx.cacheObjectContext(), cacheDataRow.key(), cacheDataRow.value(), true);
                if (isNull(qryTypeDesc))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                String tableName = qryTypeDesc.tableName();
                GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, tableName);
                if (isNull(gridH2Tbl))
                    // Tolerate - (k, v) is just not indexed.
                    continue;
                cacheSizeByTbl.computeIfAbsent(cacheCtx.cacheId(), i -> new HashMap<>()).computeIfAbsent(tableName, s -> new AtomicLong()).incrementAndGet();
            }
            PartitionUpdateCounter updateCntrAfter = locPart.dataStore().partUpdateCounter();
            if (updateCntrAfter != null && !updateCntrAfter.equals(updateCntrBefore)) {
                throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[grpName=" + grpCtx.cacheOrGroupName() + ", grpId=" + grpCtx.groupId() + ", partId=" + locPart.id() + "] changed during size " + "calculation [updCntrBefore=" + updateCntrBefore + ", updCntrAfter=" + updateCntrAfter + "]");
            }
            return new CacheSize(null, cacheSizeByTbl);
        } catch (Throwable t) {
            IgniteException cacheSizeErr = new IgniteException("Cache size calculation error [" + cacheGrpInfo(grpCtx) + ", locParId=" + partId + ", err=" + t.getMessage() + "]", t);
            error(log, cacheSizeErr);
            failCalcCacheSizeGrpIds.add(grpId);
            return new CacheSize(cacheSizeErr, null);
        } finally {
            if (reserve)
                locPart.release();
        }
    } finally {
        processedCacheSizePartitions.incrementAndGet();
        printProgressOfIndexValidationIfNeeded();
    }
}

Example 4 with GridIterator

use of org.apache.ignite.internal.util.lang.GridIterator in project ignite by apache.

the class IgniteClusterSnapshotCheckTest method testClusterSnapshotCheckFailsOnPartitionDataDiffers.

/**
 * @throws Exception If fails.
 */
@Test
public void testClusterSnapshotCheckFailsOnPartitionDataDiffers() throws Exception {
    CacheConfiguration<Integer, Value> ccfg = txCacheConfig(new CacheConfiguration<Integer, Value>(DEFAULT_CACHE_NAME)).setAffinity(new RendezvousAffinityFunction(false, 1));
    IgniteEx ignite = startGridsWithoutCache(2);
    ignite.getOrCreateCache(ccfg).put(1, new Value(new byte[2000]));
    forceCheckpoint(ignite);
    GridCacheSharedContext<?, ?> cctx = ignite.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) cctx.database();
    BinaryContext binCtx = ((CacheObjectBinaryProcessorImpl) ignite.context().cacheObjects()).binaryContext();
    GridCacheAdapter<?, ?> cache = ignite.context().cache().internalCache(dfltCacheCfg.getName());
    long partCtr = cache.context().topology().localPartition(PART_ID, NONE, false).dataStore().updateCounter();
    AtomicBoolean done = new AtomicBoolean();
    db.addCheckpointListener(new CheckpointListener() {

        @Override
        public void onMarkCheckpointBegin(Context ctx) throws IgniteCheckedException {
            // Change the cache value only at on of the cluster node to get hash conflict when the check command ends.
            if (!done.compareAndSet(false, true))
                return;
            GridIterator<CacheDataRow> it = cache.context().offheap().partitionIterator(PART_ID);
            assertTrue(it.hasNext());
            CacheDataRow row0 = it.nextX();
            AffinityTopologyVersion topVer = cctx.exchange().readyAffinityVersion();
            GridCacheEntryEx cached = cache.entryEx(row0.key(), topVer);
            byte[] bytes = new byte[2000];
            new Random().nextBytes(bytes);
            try {
                BinaryObjectImpl newVal = new BinaryObjectImpl(binCtx, binCtx.marshaller().marshal(new Value(bytes)), 0);
                boolean success = cached.initialValue(newVal, new GridCacheVersion(row0.version().topologyVersion(), row0.version().nodeOrder(), row0.version().order() + 1), null, null, TxState.NA, TxState.NA, TTL_ETERNAL, row0.expireTime(), true, topVer, DR_NONE, false, false, null);
                assertTrue(success);
                long newPartCtr = cache.context().topology().localPartition(PART_ID, NONE, false).dataStore().updateCounter();
                assertEquals(newPartCtr, partCtr);
            } catch (Exception e) {
                throw new IgniteCheckedException(e);
            }
        }

        @Override
        public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
        }

        @Override
        public void beforeCheckpointBegin(Context ctx) throws IgniteCheckedException {
        }
    });
    db.waitForCheckpoint("test-checkpoint");
    ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get();
    Path part0 = U.searchFileRecursively(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME).toPath(), getPartitionFileName(PART_ID));
    assertNotNull(part0);
    assertTrue(part0.toString(), part0.toFile().exists());
    IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get();
    StringBuilder b = new StringBuilder();
    res.print(b::append, true);
    assertTrue(F.isEmpty(res.exceptions()));
    assertContains(log, b.toString(), "The check procedure has failed, conflict partitions has been found: [counterConflicts=0, hashConflicts=1]");
}