Examples with DataEntry org.apache.ignite.internal.pagemem.wal.record.DataEntry used on opensource projects

Example 31 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class GridCacheMapEntry method mvccUpdateRowsWithPreloadInfo.

/**
 * {@inheritDoc}
 */
@Override
public GridCacheUpdateTxResult mvccUpdateRowsWithPreloadInfo(IgniteInternalTx tx, UUID affNodeId, AffinityTopologyVersion topVer, List<GridCacheEntryInfo> entries, GridCacheOperation op, MvccSnapshot mvccVer, IgniteUuid futId, int batchNum) throws IgniteCheckedException, GridCacheEntryRemovedException {
    assert mvccVer != null && MvccUtils.mvccVersionIsValid(mvccVer.coordinatorVersion(), mvccVer.counter(), mvccVer.operationCounter());
    assert !F.isEmpty(entries);
    WALPointer logPtr = null;
    ensureFreeSpace();
    CacheObject val = null;
    CacheObject oldVal = null;
    lockEntry();
    try {
        checkObsolete();
        boolean walEnabled = cctx.group().persistenceEnabled() && cctx.group().walEnabled();
        List<DataEntry> walEntries = walEnabled ? new ArrayList<>(entries.size() + 1) : Collections.EMPTY_LIST;
        // or via rebalance.
        for (int i = 0; i < entries.size(); i++) {
            GridCacheMvccEntryInfo info = (GridCacheMvccEntryInfo) entries.get(i);
            assert info.mvccTxState() == TxState.COMMITTED || MvccUtils.compare(info, mvccVer.coordinatorVersion(), mvccVer.counter()) == 0;
            assert info.newMvccTxState() == TxState.COMMITTED || MvccUtils.compareNewVersion(info, mvccVer.coordinatorVersion(), mvccVer.counter()) == 0 || info.newMvccCoordinatorVersion() == MvccUtils.MVCC_CRD_COUNTER_NA;
            boolean added = cctx.offheap().mvccUpdateRowWithPreloadInfo(this, info.value(), info.version(), info.expireTime(), info.mvccVersion(), info.newMvccVersion(), info.mvccTxState(), info.newMvccTxState());
            if (walEnabled)
                walEntries.add(toMvccDataEntry(info, tx));
            if (oldVal == null && MvccUtils.compare(info.mvccVersion(), mvccVer.coordinatorVersion(), mvccVer.counter()) != 0 && MvccUtils.compareNewVersion(info, mvccVer.coordinatorVersion(), mvccVer.counter()) == 0)
                // Old means a value before current transaction.
                oldVal = info.value();
            if (!added)
                break;
        }
        GridCacheMvccEntryInfo last = (GridCacheMvccEntryInfo) entries.get(0);
        if (walEnabled)
            Collections.reverse(walEntries);
        if (op == DELETE) {
            assert MvccUtils.compareNewVersion(last, mvccVer) == 0;
            if (walEnabled)
                walEntries.add(new MvccDataEntry(cctx.cacheId(), key, null, DELETE, tx.nearXidVersion(), last.version(), last.expireTime(), key.partition(), 0, last.mvccVersion()));
        } else {
            assert last.newMvccCoordinatorVersion() == MvccUtils.MVCC_CRD_COUNTER_NA;
            assert MvccUtils.compare(last, mvccVer) == 0;
            val = last.value();
        }
        if (walEnabled)
            logPtr = cctx.shared().wal().log(new MvccDataRecord(walEntries));
    } finally {
        if (lockedByCurrentThread()) {
            unlockEntry();
            cctx.evicts().touch(this);
        }
    }
    GridCacheUpdateTxResult res = new GridCacheUpdateTxResult(true, logPtr);
    res.newValue(val);
    res.oldValue(oldVal);
    return res;
}

Example 32 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class GridCacheDatabaseSharedManager method applyUpdatesOnRecovery.

/**
 * Apply update from some iterator and with specific filters.
 *
 * @param it WalIterator.
 * @param recPredicate Wal record filter.
 * @param entryPredicate Entry filter.
 */
public void applyUpdatesOnRecovery(@Nullable WALIterator it, IgniteBiPredicate<WALPointer, WALRecord> recPredicate, IgnitePredicate<DataEntry> entryPredicate) throws IgniteCheckedException {
    if (it == null)
        return;
    cctx.walState().runWithOutWAL(() -> {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> next = it.next();
            WALRecord rec = next.get2();
            if (!recPredicate.apply(next.get1(), rec))
                break;
            switch(rec.type()) {
                case MVCC_DATA_RECORD:
                case DATA_RECORD:
                case DATA_RECORD_V2:
                    checkpointReadLock();
                    try {
                        DataRecord dataRec = (DataRecord) rec;
                        int entryCnt = dataRec.entryCount();
                        for (int i = 0; i < entryCnt; i++) {
                            DataEntry dataEntry = dataRec.get(i);
                            if (entryPredicate.apply(dataEntry)) {
                                checkpointReadLock();
                                try {
                                    int cacheId = dataEntry.cacheId();
                                    GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
                                    if (cacheCtx != null)
                                        applyUpdate(cacheCtx, dataEntry);
                                    else if (log != null)
                                        log.warning("Cache is not started. Updates cannot be applied " + "[cacheId=" + cacheId + ']');
                                } finally {
                                    checkpointReadUnlock();
                                }
                            }
                        }
                    } catch (IgniteCheckedException e) {
                        throw new IgniteException(e);
                    } finally {
                        checkpointReadUnlock();
                    }
                    break;
                case MVCC_TX_RECORD:
                    checkpointReadLock();
                    try {
                        MvccTxRecord txRecord = (MvccTxRecord) rec;
                        byte txState = convertToTxState(txRecord.state());
                        cctx.coordinators().updateState(txRecord.mvccVersion(), txState, true);
                    } finally {
                        checkpointReadUnlock();
                    }
                    break;
                default:
            }
        }
    });
}

Example 33 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class RecoveryDebug method append.

/**
 * @param rec Data record to append.
 * @param unwrapKeyValue unwrap key and value flag.
 * @return {@code this} for convenience.
 */
public RecoveryDebug append(DataRecord rec, boolean unwrapKeyValue) {
    if (fc == null)
        return this;
    append("Data record\n");
    int entryCnt = rec.entryCount();
    for (int i = 0; i < entryCnt; i++) {
        DataEntry dataEntry = rec.get(i);
        append("\t" + dataEntry.op() + " " + dataEntry.nearXidVersion() + (unwrapKeyValue ? " " + dataEntry.key() + " " + dataEntry.value() : "") + "\n");
    }
    return this;
}

Example 34 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class IgniteWalRecoveryTest method testTxRecordsConsistency.

/**
 * Test that all DataRecord WAL records are within transaction boundaries - PREPARED and COMMITTED markers.
 *
 * @throws Exception If any fail.
 */
@Test
public void testTxRecordsConsistency() throws Exception {
    IgniteEx ignite = startGrids(3);
    ignite.cluster().state(ClusterState.ACTIVE);
    final String cacheName = "transactional";
    CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName).setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).setAffinity(new RendezvousAffinityFunction(false, 32)).setCacheMode(CacheMode.PARTITIONED).setRebalanceMode(CacheRebalanceMode.SYNC).setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC).setBackups(0);
    ignite.createCache(cacheConfiguration);
    IgniteCache<Object, Object> cache = ignite.cache(cacheName);
    GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager) sharedCtx.database();
    db.waitForCheckpoint("test");
    db.enableCheckpoints(false).get();
    // Log something to know where to start.
    WALPointer startPtr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
    final int transactions = 100;
    final int operationsPerTransaction = 40;
    Random random = new Random();
    for (int t = 1; t <= transactions; t++) {
        Transaction tx = ignite.transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
        for (int op = 0; op < operationsPerTransaction; op++) {
            int key = random.nextInt(1000) + 1;
            Object value = random.nextBoolean() ? randomString(random) + key : new BigObject(key);
            cache.put(key, value);
        }
        if (random.nextBoolean())
            tx.commit();
        else
            tx.rollback();
        if (t % 50 == 0)
            log.info("Finished transaction " + t);
    }
    Set<GridCacheVersion> activeTransactions = new HashSet<>();
    // Check that all DataRecords are within PREPARED and COMMITTED tx records.
    try (WALIterator it = sharedCtx.wal().replay(startPtr)) {
        while (it.hasNext()) {
            IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
            WALRecord rec = tup.get2();
            if (rec instanceof TxRecord) {
                TxRecord txRecord = (TxRecord) rec;
                GridCacheVersion txId = txRecord.nearXidVersion();
                switch(txRecord.state()) {
                    case PREPARED:
                        assert !activeTransactions.contains(txId) : "Transaction is already present " + txRecord;
                        activeTransactions.add(txId);
                        break;
                    case COMMITTED:
                        assert activeTransactions.contains(txId) : "No PREPARE marker for transaction " + txRecord;
                        activeTransactions.remove(txId);
                        break;
                    case ROLLED_BACK:
                        activeTransactions.remove(txId);
                        break;
                    default:
                        throw new IllegalStateException("Unknown Tx state of record " + txRecord);
                }
            } else if (rec instanceof DataRecord) {
                DataRecord dataRecord = (DataRecord) rec;
                for (int i = 0; i < dataRecord.entryCount(); i++) {
                    DataEntry entry = dataRecord.get(i);
                    GridCacheVersion txId = entry.nearXidVersion();
                    assert activeTransactions.contains(txId) : "No transaction for entry " + entry;
                }
            }
        }
    }
}

Example 35 with DataEntry

use of org.apache.ignite.internal.pagemem.wal.record.DataEntry in project ignite by apache.

the class IgniteWalReaderTest method testPutAllTxIntoTwoNodes.

/**
 * Tests transaction generation and WAL for putAll cache operation.
 *
 * @throws Exception if failed.
 */
@Test
public void testPutAllTxIntoTwoNodes() throws Exception {
    Ignite ignite = startGrid("node0");
    Ignite ignite1 = startGrid(1);
    ignite.cluster().active(true);
    Map<Object, IndexedObject> map = new TreeMap<>();
    int cntEntries = 1000;
    for (int i = 0; i < cntEntries; i++) map.put(i, new IndexedObject(i));
    ignite.cache(CACHE_NAME).putAll(map);
    ignite.cluster().active(false);
    String subfolderName1 = genDbSubfolderName(ignite, 0);
    String subfolderName2 = genDbSubfolderName(ignite1, 1);
    stopAllGrids();
    String workDir = U.defaultWorkDirectory();
    IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
    StringBuilder sb = new StringBuilder();
    Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
    IgniteInClosure<DataRecord> drHnd = dataRecord -> {
        sb.append("{");
        for (int i = 0; i < dataRecord.entryCount(); i++) {
            DataEntry entry = dataRecord.get(i);
            GridCacheOperation op = entry.op();
            Integer cnt = operationsFound.get(op);
            operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
            if (entry instanceof UnwrapDataEntry) {
                final UnwrapDataEntry entry1 = (UnwrapDataEntry) entry;
                sb.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
                final GridCacheVersion ver = entry.nearXidVersion();
                sb.append(", ");
                if (ver != null)
                    sb.append("tx=").append(ver).append(", ");
            }
        }
        sb.append("}\n");
    };
    scanIterateAndCount(factory, createIteratorParametersBuilder(workDir, subfolderName1).filesOrDirs(workDir + "/db/wal/" + subfolderName1, workDir + "/db/wal/archive/" + subfolderName1), 1, 1, null, drHnd);
    scanIterateAndCount(factory, createIteratorParametersBuilder(workDir, subfolderName2).filesOrDirs(workDir + "/db/wal/" + subfolderName2, workDir + "/db/wal/archive/" + subfolderName2), 1, 1, null, drHnd);
    Integer createsFound = operationsFound.get(CREATE);
    if (log.isInfoEnabled())
        log.info(sb.toString());
    assertTrue("Create operations should be found in log: " + operationsFound, createsFound != null && createsFound > 0);
    assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound, createsFound >= cntEntries);
}