Examples with KeyCacheObject org.apache.ignite.internal.processors.cache.KeyCacheObject used on opensource projects

Example 96 with KeyCacheObject

use of org.apache.ignite.internal.processors.cache.KeyCacheObject in project ignite by apache.

the class GridDhtAtomicCache method updateWithBatch.

/**
 * Updates locked entries using batched write-through.
 *
 * @param node Sender node.
 * @param hasNear {@code True} if originating node has near cache.
 * @param req Update request.
 * @param res Update response.
 * @param locked Locked entries.
 * @param ver Assigned version.
 * @param dhtFut Optional DHT future.
 * @param replicate Whether replication is enabled.
 * @param taskName Task name.
 * @param expiry Expiry policy.
 * @param sndPrevVal If {@code true} sends previous value to backups.
 * @return Deleted entries.
 * @throws GridCacheEntryRemovedException Should not be thrown.
 */
@SuppressWarnings("unchecked")
private DhtAtomicUpdateResult updateWithBatch(final ClusterNode node, final boolean hasNear, final GridNearAtomicAbstractUpdateRequest req, final GridNearAtomicUpdateResponse res, final List<GridDhtCacheEntry> locked, final GridCacheVersion ver, @Nullable GridDhtAtomicAbstractUpdateFuture dhtFut, final boolean replicate, final String taskName, @Nullable final IgniteCacheExpiryPolicy expiry, final boolean sndPrevVal) throws GridCacheEntryRemovedException {
    // Cannot update in batches during DR due to possible conflicts.
    assert !ctx.dr().receiveEnabled();
    // Should not request return values for putAll.
    assert !req.returnValue() || req.operation() == TRANSFORM;
    if (!F.isEmpty(req.filter()) && ctx.loadPreviousValue()) {
        try {
            reloadIfNeeded(locked);
        } catch (IgniteCheckedException e) {
            res.addFailedKeys(req.keys(), e);
            return new DhtAtomicUpdateResult();
        }
    }
    int size = req.size();
    Map<KeyCacheObject, CacheObject> putMap = null;
    Map<KeyCacheObject, EntryProcessor<Object, Object, Object>> entryProcessorMap = null;
    Collection<KeyCacheObject> rmvKeys = null;
    List<CacheObject> writeVals = null;
    DhtAtomicUpdateResult updRes = new DhtAtomicUpdateResult();
    List<GridDhtCacheEntry> filtered = new ArrayList<>(size);
    GridCacheOperation op = req.operation();
    GridCacheReturn invokeRes = null;
    int firstEntryIdx = 0;
    boolean intercept = ctx.config().getInterceptor() != null;
    for (int i = 0; i < locked.size(); i++) {
        GridDhtCacheEntry entry = locked.get(i);
        try {
            if (!checkFilter(entry, req, res)) {
                if (expiry != null && entry.hasValue()) {
                    long ttl = expiry.forAccess();
                    if (ttl != CU.TTL_NOT_CHANGED) {
                        entry.updateTtl(null, ttl);
                        expiry.ttlUpdated(entry.key(), entry.version(), entry.readers());
                    }
                }
                if (log.isDebugEnabled())
                    log.debug("Entry did not pass the filter (will skip write) [entry=" + entry + ", filter=" + Arrays.toString(req.filter()) + ", res=" + res + ']');
                if (hasNear)
                    res.addSkippedIndex(i);
                firstEntryIdx++;
                continue;
            }
            if (op == TRANSFORM) {
                EntryProcessor<Object, Object, Object> entryProcessor = req.entryProcessor(i);
                CacheObject old = entry.innerGet(ver, null, /*read through*/
                true, /*metrics*/
                true, /*event*/
                true, req.subjectId(), entryProcessor, taskName, null, req.keepBinary());
                Object oldVal = null;
                Object updatedVal = null;
                CacheInvokeEntry<Object, Object> invokeEntry = new CacheInvokeEntry(entry.key(), old, entry.version(), req.keepBinary(), entry);
                CacheObject updated = null;
                if (invokeRes == null)
                    invokeRes = new GridCacheReturn(node.isLocal());
                CacheInvokeResult curInvokeRes = null;
                boolean validation = false;
                try {
                    Object computed = entryProcessor.process(invokeEntry, req.invokeArguments());
                    if (computed != null) {
                        computed = ctx.unwrapTemporary(computed);
                        curInvokeRes = CacheInvokeResult.fromResult(computed);
                    }
                    if (!invokeEntry.modified())
                        continue;
                    else {
                        updatedVal = ctx.unwrapTemporary(invokeEntry.getValue());
                        updated = ctx.toCacheObject(updatedVal);
                        validation = true;
                        ctx.validateKeyAndValue(entry.key(), updated);
                    }
                } catch (Exception e) {
                    curInvokeRes = CacheInvokeResult.fromError(e);
                    updated = old;
                    if (validation) {
                        res.addSkippedIndex(i);
                        continue;
                    }
                } finally {
                    if (curInvokeRes != null) {
                        invokeRes.addEntryProcessResult(ctx, entry.key(), invokeEntry.key(), curInvokeRes.result(), curInvokeRes.error(), req.keepBinary());
                    }
                }
                if (updated == null) {
                    if (intercept) {
                        CacheLazyEntry e = new CacheLazyEntry(ctx, entry.key(), invokeEntry.key(), old, oldVal, req.keepBinary());
                        IgniteBiTuple<Boolean, ?> interceptorRes = ctx.config().getInterceptor().onBeforeRemove(e);
                        if (ctx.cancelRemove(interceptorRes))
                            continue;
                    }
                    // Update previous batch.
                    if (putMap != null) {
                        dhtFut = updatePartialBatch(hasNear, firstEntryIdx, filtered, ver, node, writeVals, putMap, null, entryProcessorMap, dhtFut, req, res, replicate, updRes, taskName, expiry, sndPrevVal);
                        firstEntryIdx = i;
                        putMap = null;
                        writeVals = null;
                        entryProcessorMap = null;
                        filtered = new ArrayList<>();
                    }
                    // Start collecting new batch.
                    if (rmvKeys == null)
                        rmvKeys = new ArrayList<>(size);
                    rmvKeys.add(entry.key());
                } else {
                    if (intercept) {
                        CacheLazyEntry e = new CacheLazyEntry(ctx, entry.key(), invokeEntry.key(), old, oldVal, req.keepBinary());
                        Object val = ctx.config().getInterceptor().onBeforePut(e, updatedVal);
                        if (val == null)
                            continue;
                        updated = ctx.toCacheObject(ctx.unwrapTemporary(val));
                    }
                    // Update previous batch.
                    if (rmvKeys != null) {
                        dhtFut = updatePartialBatch(hasNear, firstEntryIdx, filtered, ver, node, null, null, rmvKeys, entryProcessorMap, dhtFut, req, res, replicate, updRes, taskName, expiry, sndPrevVal);
                        firstEntryIdx = i;
                        rmvKeys = null;
                        entryProcessorMap = null;
                        filtered = new ArrayList<>();
                    }
                    if (putMap == null) {
                        putMap = new LinkedHashMap<>(size, 1.0f);
                        writeVals = new ArrayList<>(size);
                    }
                    putMap.put(entry.key(), updated);
                    writeVals.add(updated);
                }
                if (entryProcessorMap == null)
                    entryProcessorMap = new HashMap<>();
                entryProcessorMap.put(entry.key(), entryProcessor);
            } else if (op == UPDATE) {
                CacheObject updated = req.value(i);
                if (intercept) {
                    CacheObject old = entry.innerGet(null, null, /*read through*/
                    ctx.loadPreviousValue(), /*metrics*/
                    true, /*event*/
                    true, req.subjectId(), null, taskName, null, req.keepBinary());
                    Object val = ctx.config().getInterceptor().onBeforePut(new CacheLazyEntry(ctx, entry.key(), old, req.keepBinary()), ctx.unwrapBinaryIfNeeded(updated, req.keepBinary(), false));
                    if (val == null)
                        continue;
                    updated = ctx.toCacheObject(ctx.unwrapTemporary(val));
                }
                assert updated != null;
                ctx.validateKeyAndValue(entry.key(), updated);
                if (putMap == null) {
                    putMap = new LinkedHashMap<>(size, 1.0f);
                    writeVals = new ArrayList<>(size);
                }
                putMap.put(entry.key(), updated);
                writeVals.add(updated);
            } else {
                assert op == DELETE;
                if (intercept) {
                    CacheObject old = entry.innerGet(null, null, /*read through*/
                    ctx.loadPreviousValue(), /*metrics*/
                    true, /*event*/
                    true, req.subjectId(), null, taskName, null, req.keepBinary());
                    IgniteBiTuple<Boolean, ?> interceptorRes = ctx.config().getInterceptor().onBeforeRemove(new CacheLazyEntry(ctx, entry.key(), old, req.keepBinary()));
                    if (ctx.cancelRemove(interceptorRes))
                        continue;
                }
                if (rmvKeys == null)
                    rmvKeys = new ArrayList<>(size);
                rmvKeys.add(entry.key());
            }
            filtered.add(entry);
        } catch (IgniteCheckedException e) {
            res.addFailedKey(entry.key(), e);
        }
    }
    // Store final batch.
    if (putMap != null || rmvKeys != null) {
        dhtFut = updatePartialBatch(hasNear, firstEntryIdx, filtered, ver, node, writeVals, putMap, rmvKeys, entryProcessorMap, dhtFut, req, res, replicate, updRes, taskName, expiry, sndPrevVal);
    } else
        assert filtered.isEmpty();
    updRes.dhtFuture(dhtFut);
    updRes.returnValue(invokeRes);
    return updRes;
}

Example 97 with KeyCacheObject

use of org.apache.ignite.internal.processors.cache.KeyCacheObject in project ignite by apache.

the class GridDhtAtomicCache method updateSingle.

/**
 * Updates locked entries one-by-one.
 *
 * @param nearNode Originating node.
 * @param hasNear {@code True} if originating node has near cache.
 * @param req Update request.
 * @param res Update response.
 * @param locked Locked entries.
 * @param ver Assigned update version.
 * @param dhtFut Optional DHT future.
 * @param replicate Whether DR is enabled for that cache.
 * @param taskName Task name.
 * @param expiry Expiry policy.
 * @param sndPrevVal If {@code true} sends previous value to backups.
 * @return Return value.
 * @throws GridCacheEntryRemovedException Should be never thrown.
 */
private DhtAtomicUpdateResult updateSingle(ClusterNode nearNode, boolean hasNear, GridNearAtomicAbstractUpdateRequest req, GridNearAtomicUpdateResponse res, List<GridDhtCacheEntry> locked, GridCacheVersion ver, @Nullable GridDhtAtomicAbstractUpdateFuture dhtFut, boolean replicate, String taskName, @Nullable IgniteCacheExpiryPolicy expiry, boolean sndPrevVal) throws GridCacheEntryRemovedException {
    GridCacheReturn retVal = null;
    Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
    AffinityTopologyVersion topVer = req.topologyVersion();
    boolean intercept = ctx.config().getInterceptor() != null;
    AffinityAssignment affAssignment = ctx.affinity().assignment(topVer);
    // Avoid iterator creation.
    for (int i = 0; i < req.size(); i++) {
        KeyCacheObject k = req.key(i);
        GridCacheOperation op = req.operation();
        // No GridCacheEntryRemovedException can be thrown.
        try {
            GridDhtCacheEntry entry = locked.get(i);
            GridCacheVersion newConflictVer = req.conflictVersion(i);
            long newConflictTtl = req.conflictTtl(i);
            long newConflictExpireTime = req.conflictExpireTime(i);
            assert !(newConflictVer instanceof GridCacheVersionEx) : newConflictVer;
            Object writeVal = op == TRANSFORM ? req.entryProcessor(i) : req.writeValue(i);
            // Get readers before innerUpdate (reader cleared after remove).
            GridDhtCacheEntry.ReaderId[] readers = entry.readersLocked();
            GridCacheUpdateAtomicResult updRes = entry.innerUpdate(ver, nearNode.id(), locNodeId, op, writeVal, req.invokeArguments(), writeThrough() && !req.skipStore(), !req.skipStore(), sndPrevVal || req.returnValue(), req.keepBinary(), expiry, /*event*/
            true, /*metrics*/
            true, /*primary*/
            true, /*verCheck*/
            false, topVer, req.filter(), replicate ? DR_PRIMARY : DR_NONE, newConflictTtl, newConflictExpireTime, newConflictVer, /*conflictResolve*/
            true, intercept, req.subjectId(), taskName, /*prevVal*/
            null, /*updateCntr*/
            null, dhtFut);
            if (dhtFut != null) {
                if (updRes.sendToDht()) {
                    // Send to backups even in case of remove-remove scenarios.
                    GridCacheVersionConflictContext<?, ?> conflictCtx = updRes.conflictResolveResult();
                    if (conflictCtx == null)
                        newConflictVer = null;
                    else if (conflictCtx.isMerge())
                        // Conflict version is discarded in case of merge.
                        newConflictVer = null;
                    EntryProcessor<Object, Object, Object> entryProcessor = null;
                    dhtFut.addWriteEntry(affAssignment, entry, updRes.newValue(), entryProcessor, updRes.newTtl(), updRes.conflictExpireTime(), newConflictVer, sndPrevVal, updRes.oldValue(), updRes.updateCounter());
                    if (readers != null)
                        dhtFut.addNearWriteEntries(nearNode, readers, entry, updRes.newValue(), entryProcessor, updRes.newTtl(), updRes.conflictExpireTime());
                } else {
                    if (log.isDebugEnabled())
                        log.debug("Entry did not pass the filter or conflict resolution (will skip write) " + "[entry=" + entry + ", filter=" + Arrays.toString(req.filter()) + ']');
                }
            }
            if (hasNear) {
                if (updRes.sendToDht()) {
                    if (!ctx.affinity().partitionBelongs(nearNode, entry.partition(), topVer)) {
                        // If put the same value as in request then do not need to send it back.
                        if (op == TRANSFORM || writeVal != updRes.newValue()) {
                            res.addNearValue(i, updRes.newValue(), updRes.newTtl(), updRes.conflictExpireTime());
                        } else
                            res.addNearTtl(i, updRes.newTtl(), updRes.conflictExpireTime());
                        if (updRes.newValue() != null) {
                            IgniteInternalFuture<Boolean> f = entry.addReader(nearNode.id(), req.messageId(), topVer);
                            assert f == null : f;
                        }
                    } else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
                        // Reader became primary or backup.
                        entry.removeReader(nearNode.id(), req.messageId());
                    } else
                        res.addSkippedIndex(i);
                } else
                    res.addSkippedIndex(i);
            }
            if (updRes.removeVersion() != null) {
                if (deleted == null)
                    deleted = new ArrayList<>(req.size());
                deleted.add(F.t(entry, updRes.removeVersion()));
            }
            if (op == TRANSFORM) {
                assert !req.returnValue();
                IgniteBiTuple<Object, Exception> compRes = updRes.computedResult();
                if (compRes != null && (compRes.get1() != null || compRes.get2() != null)) {
                    if (retVal == null)
                        retVal = new GridCacheReturn(nearNode.isLocal());
                    retVal.addEntryProcessResult(ctx, k, null, compRes.get1(), compRes.get2(), req.keepBinary());
                }
            } else {
                // Create only once.
                if (retVal == null) {
                    CacheObject ret = updRes.oldValue();
                    retVal = new GridCacheReturn(ctx, nearNode.isLocal(), req.keepBinary(), req.returnValue() ? ret : null, updRes.success());
                }
            }
        } catch (IgniteCheckedException e) {
            res.addFailedKey(k, e);
        }
    }
    return new DhtAtomicUpdateResult(retVal, deleted, dhtFut);
}

Example 98 with KeyCacheObject

use of org.apache.ignite.internal.processors.cache.KeyCacheObject in project ignite by apache.

the class IgnitePdsCheckpointSimulationWithRealCpDisabledTest method testDataWalEntries.

/**
 * @throws Exception if failed.
 */
public void testDataWalEntries() throws Exception {
    IgniteEx ig = startGrid(0);
    ig.active(true);
    GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
    GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(cacheName).context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    IgniteWriteAheadLogManager wal = sharedCtx.wal();
    assertTrue(wal.isAlwaysWriteFullPages());
    db.enableCheckpoints(false).get();
    final int cnt = 10;
    List<DataEntry> entries = new ArrayList<>(cnt);
    for (int i = 0; i < cnt; i++) {
        GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
        KeyCacheObject key = cctx.toCacheKeyObject(i);
        CacheObject val = null;
        if (op != GridCacheOperation.DELETE)
            val = cctx.toCacheObject("value-" + i);
        entries.add(new DataEntry(cctx.cacheId(), key, val, op, null, cctx.versions().next(), 0L, cctx.affinity().partition(i), i));
    }
    UUID cpId = UUID.randomUUID();
    WALPointer start = wal.log(new CheckpointRecord(cpId, null));
    wal.fsync(start);
    for (DataEntry entry : entries) wal.log(new DataRecord(entry));
    // Data will not be written to the page store.
    stopAllGrids();
    ig = startGrid(0);
    ig.active(true);
    sharedCtx = ig.context().cache().context();
    cctx = sharedCtx.cache().cache(cacheName).context();
    db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    wal = sharedCtx.wal();
    db.enableCheckpoints(false).get();
    try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
        IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
        assert cpRecordTup.get2() instanceof CheckpointRecord;
        assertEquals(start, cpRecordTup.get1());
        CheckpointRecord cpRec = (CheckpointRecord) cpRecordTup.get2();
        assertEquals(cpId, cpRec.checkpointId());
        assertNull(cpRec.checkpointMark());
        assertFalse(cpRec.end());
        int idx = 0;
        CacheObjectContext coctx = cctx.cacheObjectContext();
        while (idx < entries.size()) {
            IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
            assert dataRecTup.get2() instanceof DataRecord;
            DataRecord dataRec = (DataRecord) dataRecTup.get2();
            DataEntry entry = entries.get(idx);
            assertEquals(1, dataRec.writeEntries().size());
            DataEntry readEntry = dataRec.writeEntries().get(0);
            assertEquals(entry.cacheId(), readEntry.cacheId());
            assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
            assertEquals(entry.op(), readEntry.op());
            if (entry.op() == GridCacheOperation.UPDATE)
                assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
            else
                assertNull(entry.value());
            assertEquals(entry.writeVersion(), readEntry.writeVersion());
            assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
            assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
            idx++;
        }
    }
}

Example 99 with KeyCacheObject

use of org.apache.ignite.internal.processors.cache.KeyCacheObject in project ignite by apache.

the class IgniteWalReaderTest method iterateAndCount.

/**
 * Iterates on records and closes iterator.
 *
 * @param walIter iterator to count, will be closed.
 * @param touchEntries access data within entries.
 * @return count of records.
 * @throws IgniteCheckedException if failed to iterate.
 */
private int iterateAndCount(WALIterator walIter, boolean touchEntries) throws IgniteCheckedException {
    int cnt = 0;
    try (WALIterator it = walIter) {
        while (it.hasNextX()) {
            final IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
            final WALRecord walRecord = next.get2();
            if (touchEntries && walRecord.type() == WALRecord.RecordType.DATA_RECORD) {
                final DataRecord record = (DataRecord) walRecord;
                for (DataEntry entry : record.writeEntries()) {
                    final KeyCacheObject key = entry.key();
                    final CacheObject val = entry.value();
                    if (dumpRecords)
                        log.info("Op: " + entry.op() + ", Key: " + key + ", Value: " + val);
                }
            }
            if (dumpRecords)
                log.info("Record: " + walRecord);
            cnt++;
        }
    }
    return cnt;
}

Example 100 with KeyCacheObject

use of org.apache.ignite.internal.processors.cache.KeyCacheObject in project ignite by apache.

the class SchemaIndexCacheVisitorImpl method processPartition.

/**
 * Process partition.
 *
 * @param part Partition.
 * @param clo Index closure.
 * @throws IgniteCheckedException If failed.
 */
private void processPartition(GridDhtLocalPartition part, SchemaIndexCacheVisitorClosure clo) throws IgniteCheckedException {
    checkCancelled();
    boolean reserved = false;
    if (part != null && part.state() != EVICTED)
        reserved = (part.state() == OWNING || part.state() == RENTING) && part.reserve();
    if (!reserved)
        return;
    try {
        GridCursor<? extends CacheDataRow> cursor = part.dataStore().cursor(cctx.cacheId(), null, null, CacheDataRowAdapter.RowData.KEY_ONLY);
        boolean locked = false;
        try {
            int cntr = 0;
            while (cursor.next() && !stop) {
                KeyCacheObject key = cursor.get().key();
                if (!locked) {
                    cctx.shared().database().checkpointReadLock();
                    locked = true;
                }
                processKey(key, clo);
                if (++cntr % BATCH_SIZE == 0) {
                    cctx.shared().database().checkpointReadUnlock();
                    locked = false;
                }
                if (part.state() == RENTING)
                    break;
            }
        } finally {
            if (locked)
                cctx.shared().database().checkpointReadUnlock();
        }
    } finally {
        part.release();
    }
}