use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.
the class GridCacheDatabaseSharedManager method tailPointer.
/**
* Calculates tail pointer for WAL at the end of logical recovery.
*
* @param logicalState State after logical recovery.
* @return Tail pointer.
* @throws IgniteCheckedException If failed.
*/
private WALPointer tailPointer(RestoreLogicalState logicalState) throws IgniteCheckedException {
// Should flush all data in buffers before read last WAL pointer.
// Iterator read records only from files.
WALPointer lastFlushPtr = cctx.wal().flush(null, true);
// We must return null for NULL_PTR record, because FileWriteAheadLogManager.resumeLogging
// can't write header without that condition.
WALPointer lastReadPtr = logicalState.lastReadRecordPointer();
if (lastFlushPtr != null && lastReadPtr == null)
return lastFlushPtr;
if (lastFlushPtr == null && lastReadPtr != null)
return lastReadPtr;
if (lastFlushPtr != null && lastReadPtr != null)
return lastReadPtr.compareTo(lastFlushPtr) >= 0 ? lastReadPtr : lastFlushPtr;
return null;
}
use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.
the class IgniteTxLocalAdapter method userCommit.
/**
* {@inheritDoc}
*/
@Override
public void userCommit() throws IgniteCheckedException {
TransactionState state = state();
if (state != COMMITTING) {
if (remainingTime() == -1)
throw new IgniteTxTimeoutCheckedException("Transaction timed out: " + this);
setRollbackOnly();
throw new IgniteCheckedException("Invalid transaction state for commit [state=" + state + ", tx=" + this + ']');
}
checkValid();
Collection<IgniteTxEntry> commitEntries = (near() || cctx.snapshot().needTxReadLogging()) ? allEntries() : writeEntries();
boolean empty = F.isEmpty(commitEntries) && !queryEnlisted();
// locks on backup nodes.
if (!empty || colocated())
cctx.tm().addCommittedTx(this);
if (!empty) {
batchStoreCommit(writeEntries());
WALPointer ptr = null;
IgniteCheckedException err = null;
cctx.database().checkpointReadLock();
try {
cctx.tm().txContext(this);
AffinityTopologyVersion topVer = topologyVersion();
TxCounters txCounters = txCounters(false);
/*
* Commit to cache. Note that for 'near' transaction we loop through all the entries.
*/
for (IgniteTxEntry txEntry : commitEntries) {
GridCacheContext cacheCtx = txEntry.context();
GridDrType drType = cacheCtx.isDrEnabled() ? DR_PRIMARY : DR_NONE;
UUID nodeId = txEntry.nodeId() == null ? this.nodeId : txEntry.nodeId();
while (true) {
try {
GridCacheEntryEx cached = txEntry.cached();
// transaction manager to make sure locks are held.
if (!evictNearEntry(txEntry, false)) {
if (cacheCtx.isNear() && cacheCtx.dr().receiveEnabled()) {
cached.markObsolete(xidVer);
break;
}
if (cached.detached())
break;
boolean updateNearCache = updateNearCache(cacheCtx, txEntry.key(), topVer);
boolean metrics = true;
if (!updateNearCache && cacheCtx.isNear() && txEntry.locallyMapped())
metrics = false;
boolean evt = !isNearLocallyMapped(txEntry, false);
if (!F.isEmpty(txEntry.entryProcessors()) || !F.isEmpty(txEntry.filters()))
txEntry.cached().unswap(false);
IgniteBiTuple<GridCacheOperation, CacheObject> res = applyTransformClosures(txEntry, true, null);
GridCacheVersion dhtVer = null;
// backup remote transaction completes.
if (cacheCtx.isNear()) {
if (txEntry.op() == CREATE || txEntry.op() == UPDATE || txEntry.op() == DELETE || txEntry.op() == TRANSFORM)
dhtVer = txEntry.dhtVersion();
if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) && txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
if (expiry != null) {
txEntry.cached().unswap(false);
Duration duration = cached.hasValue() ? expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
txEntry.ttl(CU.toTtl(duration));
}
}
}
GridCacheOperation op = res.get1();
CacheObject val = res.get2();
// Deal with conflicts.
GridCacheVersion explicitVer = txEntry.conflictVersion() != null ? txEntry.conflictVersion() : writeVersion();
if ((op == CREATE || op == UPDATE) && txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
if (expiry != null) {
Duration duration = cached.hasValue() ? expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
long ttl = CU.toTtl(duration);
txEntry.ttl(ttl);
if (ttl == CU.TTL_ZERO)
op = DELETE;
}
}
boolean conflictNeedResolve = cacheCtx.conflictNeedResolve();
GridCacheVersionConflictContext<?, ?> conflictCtx = null;
if (conflictNeedResolve) {
IgniteBiTuple<GridCacheOperation, GridCacheVersionConflictContext> conflictRes = conflictResolve(op, txEntry, val, explicitVer, cached);
assert conflictRes != null;
conflictCtx = conflictRes.get2();
if (conflictCtx.isUseOld())
op = NOOP;
else if (conflictCtx.isUseNew()) {
txEntry.ttl(conflictCtx.ttl());
txEntry.conflictExpireTime(conflictCtx.expireTime());
} else {
assert conflictCtx.isMerge();
op = conflictRes.get1();
val = txEntry.context().toCacheObject(conflictCtx.mergeValue());
explicitVer = writeVersion();
txEntry.ttl(conflictCtx.ttl());
txEntry.conflictExpireTime(conflictCtx.expireTime());
}
} else
// Nullify explicit version so that innerSet/innerRemove will work as usual.
explicitVer = null;
if (sndTransformedVals || conflictNeedResolve) {
assert sndTransformedVals && cacheCtx.isReplicated() || conflictNeedResolve;
txEntry.value(val, true, false);
txEntry.op(op);
txEntry.entryProcessors(null);
txEntry.conflictVersion(explicitVer);
}
if (dhtVer == null)
dhtVer = explicitVer != null ? explicitVer : writeVersion();
if (op == CREATE || op == UPDATE) {
assert val != null : txEntry;
GridCacheUpdateTxResult updRes = cached.innerSet(this, eventNodeId(), txEntry.nodeId(), val, false, false, txEntry.ttl(), evt, metrics, txEntry.keepBinary(), txEntry.hasOldValue(), txEntry.oldValue(), topVer, null, cached.detached() ? DR_NONE : drType, txEntry.conflictExpireTime(), cached.isNear() ? null : explicitVer, resolveTaskName(), dhtVer, null);
if (updRes.success())
txEntry.updateCounter(updRes.updateCounter());
if (updRes.loggedPointer() != null)
ptr = updRes.loggedPointer();
if (updRes.success() && updateNearCache) {
final CacheObject val0 = val;
final boolean metrics0 = metrics;
final GridCacheVersion dhtVer0 = dhtVer;
updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerSet(null, eventNodeId(), nodeId, val0, false, false, txEntry.ttl(), false, metrics0, txEntry.keepBinary(), txEntry.hasOldValue(), txEntry.oldValue(), topVer, CU.empty0(), DR_NONE, txEntry.conflictExpireTime(), null, resolveTaskName(), dhtVer0, null));
}
} else if (op == DELETE) {
GridCacheUpdateTxResult updRes = cached.innerRemove(this, eventNodeId(), txEntry.nodeId(), false, evt, metrics, txEntry.keepBinary(), txEntry.hasOldValue(), txEntry.oldValue(), topVer, null, cached.detached() ? DR_NONE : drType, cached.isNear() ? null : explicitVer, resolveTaskName(), dhtVer, null);
if (updRes.success())
txEntry.updateCounter(updRes.updateCounter());
if (updRes.loggedPointer() != null)
ptr = updRes.loggedPointer();
if (updRes.success() && updateNearCache) {
final boolean metrics0 = metrics;
final GridCacheVersion dhtVer0 = dhtVer;
updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerRemove(null, eventNodeId(), nodeId, false, false, metrics0, txEntry.keepBinary(), txEntry.hasOldValue(), txEntry.oldValue(), topVer, CU.empty0(), DR_NONE, null, resolveTaskName(), dhtVer0, null));
}
} else if (op == RELOAD) {
cached.innerReload();
if (updateNearCache)
updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerReload());
} else if (op == READ) {
CacheGroupContext grp = cacheCtx.group();
if (grp.persistenceEnabled() && grp.walEnabled() && cctx.snapshot().needTxReadLogging()) {
ptr = cctx.wal().log(new DataRecord(new DataEntry(cacheCtx.cacheId(), txEntry.key(), val, op, nearXidVersion(), writeVersion(), 0, txEntry.key().partition(), txEntry.updateCounter(), DataEntry.flags(CU.txOnPrimary(this)))));
}
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
if (expiry != null) {
Duration duration = expiry.getExpiryForAccess();
if (duration != null)
cached.updateTtl(null, CU.toTtl(duration));
}
if (log.isDebugEnabled())
log.debug("Ignoring READ entry when committing: " + txEntry);
} else {
assert ownsLock(txEntry.cached()) : "Transaction does not own lock for group lock entry during commit [tx=" + this + ", txEntry=" + txEntry + ']';
if (conflictCtx == null || !conflictCtx.isUseOld()) {
if (txEntry.ttl() != CU.TTL_NOT_CHANGED)
cached.updateTtl(null, txEntry.ttl());
}
if (log.isDebugEnabled())
log.debug("Ignoring NOOP entry when committing: " + txEntry);
}
}
// if an entry is obsolete).
if (txEntry.op() != READ)
checkCommitLocks(cached);
// Break out of while loop.
break;
}// If entry cached within transaction got removed.
catch (GridCacheEntryRemovedException ignored) {
if (log.isDebugEnabled())
log.debug("Got removed entry during transaction commit (will retry): " + txEntry);
txEntry.cached(entryEx(cacheCtx, txEntry.txKey(), topologyVersion()));
}
}
}
if (!txState.mvccEnabled() && txCounters != null) {
cctx.tm().txHandler().applyPartitionsUpdatesCounters(txCounters.updateCounters());
for (IgniteTxEntry entry : commitEntries) {
if (entry.cqNotifyClosure() != null)
entry.cqNotifyClosure().applyx();
}
}
// Apply cache sizes only for primary nodes. Update counters were applied on prepare state.
applyTxSizes();
cctx.mvccCaching().onTxFinished(this, true);
if (ptr != null)
cctx.wal().flush(ptr, false);
} catch (Throwable ex) {
// We are about to initiate transaction rollback when tx has started to committing.
// Need to remove version from committed list.
cctx.tm().removeCommittedTx(this);
if (X.hasCause(ex, NodeStoppingException.class)) {
U.warn(log, "Failed to commit transaction, node is stopping [tx=" + CU.txString(this) + ", err=" + ex + ']');
boolean persistenceEnabled = CU.isPersistenceEnabled(cctx.kernalContext().config());
if (persistenceEnabled) {
GridCacheDatabaseSharedManager dbManager = (GridCacheDatabaseSharedManager) cctx.database();
dbManager.getCheckpointer().skipCheckpointOnNodeStop(true);
}
throw ex;
}
err = heuristicException(ex);
COMMIT_ERR_UPD.compareAndSet(this, null, err);
state(UNKNOWN);
try {
uncommit();
} catch (Throwable e) {
err.addSuppressed(e);
}
throw err;
} finally {
cctx.database().checkpointReadUnlock();
cctx.tm().resetContext();
}
}
// Do not unlock transaction entries if one-phase commit.
if (!onePhaseCommit()) {
if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
// Unlock all locks.
cctx.tm().commitTx(this);
boolean needsCompletedVersions = needsCompletedVersions();
assert !needsCompletedVersions || completedBase != null;
assert !needsCompletedVersions || committedVers != null;
assert !needsCompletedVersions || rolledbackVers != null;
}
}
}
use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.
the class IgniteTxHandler method applyPartitionsUpdatesCounters.
/**
* Applies partition counter updates for transactions.
* <p>
* Called after entries are written to WAL on commit or during rollback to close gaps in update counter sequence.
* <p>
* On rollback counters should be applied on the primary only after backup nodes, otherwise if the primary fail
* before sending rollback requests to backups remote transactions can be committed by recovery protocol and
* partition consistency will not be restored when primary returns to the grid because RollbackRecord was written
* (actual for persistent mode only).
*
* @param counters Counter values to be updated.
* @param rollback {@code True} if applied during rollbacks.
* @param rollbackOnPrimary {@code True} if rollback happens on primary node. Passed to CQ engine.
*/
public void applyPartitionsUpdatesCounters(Iterable<PartitionUpdateCountersMessage> counters, boolean rollback, boolean rollbackOnPrimary) throws IgniteCheckedException {
if (counters == null)
return;
WALPointer ptr = null;
try {
for (PartitionUpdateCountersMessage counter : counters) {
GridCacheContext ctx0 = ctx.cacheContext(counter.cacheId());
GridDhtPartitionTopology top = ctx0.topology();
AffinityTopologyVersion topVer = top.readyTopologyVersion();
assert top != null;
for (int i = 0; i < counter.size(); i++) {
boolean invalid = false;
try {
GridDhtLocalPartition part = top.localPartition(counter.partition(i));
if (part != null && part.reserve()) {
try {
if (part.state() != RENTING) {
// Check is actual only for backup node.
long start = counter.initialCounter(i);
long delta = counter.updatesCount(i);
boolean updated = part.updateCounter(start, delta);
// Need to log rolled back range for logical recovery.
if (updated && rollback) {
CacheGroupContext grpCtx = part.group();
if (grpCtx.persistenceEnabled() && grpCtx.walEnabled() && !grpCtx.mvccEnabled()) {
RollbackRecord rec = new RollbackRecord(grpCtx.groupId(), part.id(), start, delta);
ptr = ctx.wal().log(rec);
}
for (int cntr = 1; cntr <= delta; cntr++) {
ctx0.continuousQueries().skipUpdateCounter(null, part.id(), start + cntr, topVer, rollbackOnPrimary);
}
}
} else
invalid = true;
} finally {
part.release();
}
} else
invalid = true;
} catch (GridDhtInvalidPartitionException e) {
invalid = true;
}
if (log.isDebugEnabled() && invalid) {
log.debug("Received partition update counters message for invalid partition, ignoring: " + "[cacheId=" + counter.cacheId() + ", part=" + counter.partition(i) + ']');
}
}
}
} finally {
if (ptr != null)
ctx.wal().flush(ptr, false);
}
}
use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.
the class CdcConsumerState method load.
/**
* Loads CDC state from file.
* @return Saved state.
*/
public WALPointer load() {
if (!Files.exists(state))
return null;
try (FileChannel ch = FileChannel.open(state, StandardOpenOption.READ)) {
ByteBuffer buf = ByteBuffer.allocate(POINTER_SIZE);
ch.read(buf);
buf.flip();
long idx = buf.getLong();
int offset = buf.getInt();
int length = buf.getInt();
return new WALPointer(idx, offset, length);
} catch (IOException e) {
throw new IgniteException("Failed to read state [file=" + state + ']', e);
}
}
use of org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer in project ignite by apache.
the class MetaStorage method removeData.
/**
*/
public void removeData(String key) throws IgniteCheckedException {
if (!readOnly) {
WALPointer ptr;
synchronized (this) {
MetastorageDataRow oldRow = tree.findOne(new MetastorageSearchRow(key));
if (oldRow == null)
return;
ptr = wal.log(new MetastoreDataRecord(key, null));
tree.removex(oldRow);
tree.rowStore().removeRow(oldRow.link());
if (oldRow.keyLink() != 0L)
tree.rowStore().removeRow(oldRow.keyLink());
}
wal.flush(ptr, false);
}
}
Aggregations