use of org.apache.geode.internal.cache.persistence.DiskRecoveryStore in project geode by apache.
the class Oplog method readKrf.
private boolean readKrf(OplogEntryIdSet deletedIds, boolean recoverValues, boolean recoverValuesSync, Set<Oplog> oplogsNeedingValueRecovery, boolean latestOplog) {
File f = new File(this.diskFile.getPath() + KRF_FILE_EXT);
if (!f.exists()) {
return false;
}
if (!getParent().getDiskInitFile().hasKrf(this.oplogId)) {
logger.info(LocalizedMessage.create(LocalizedStrings.Oplog_REMOVING_INCOMPLETE_KRF, new Object[] { f.getName(), this.oplogId, getParent().getName() }));
f.delete();
}
// Set krfCreated to true since we have a krf.
this.krfCreated.set(true);
// so that we don't try to recreate the krf.
if (recoverValuesSync) {
return false;
}
FileInputStream fis;
try {
fis = new FileInputStream(f);
} catch (FileNotFoundException ignore) {
return false;
}
try {
if (getParent().isOffline() && !getParent().FORCE_KRF_RECOVERY) {
return false;
}
logger.info(LocalizedMessage.create(LocalizedStrings.DiskRegion_RECOVERING_OPLOG_0_1_2, new Object[] { toString(), f.getAbsolutePath(), getParent().getName() }));
this.recoverNewEntryId = DiskStoreImpl.INVALID_ID;
this.recoverModEntryId = DiskStoreImpl.INVALID_ID;
this.recoverModEntryIdHWM = DiskStoreImpl.INVALID_ID;
long oplogKeyIdHWM = DiskStoreImpl.INVALID_ID;
int krfEntryCount = 0;
DataInputStream dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
final Version version = getProductVersionIfOld();
final ByteArrayDataInput in = new ByteArrayDataInput();
try {
try {
validateOpcode(dis, OPLOG_MAGIC_SEQ_ID);
readOplogMagicSeqRecord(dis, f, OPLOG_TYPE.KRF);
validateOpcode(dis, OPLOG_DISK_STORE_ID);
readDiskStoreRecord(dis, f);
} catch (DiskAccessException ignore) {
// Failed to read the file. There are two possibilities. Either this
// file is in old format which does not have a magic seq in the
// beginning or this is not a valid file at all. Try reading it as a
// file in old format
fis.close();
fis = new FileInputStream(f);
dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
readDiskStoreRecord(dis, f);
} catch (IllegalStateException ignore) {
// Failed to read the file. There are two possibilities. Either this
// is in new format which has a magic seq in the beginning or this is
// not a valid file at all
fis.close();
fis = new FileInputStream(f);
dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
readDiskStoreRecord(dis, f);
}
readGemfireVersionRecord(dis, f);
readTotalCountRecord(dis, f);
readRVVRecord(dis, f, false, latestOplog);
long lastOffset = 0;
byte[] keyBytes = DataSerializer.readByteArray(dis);
while (keyBytes != null) {
byte userBits = dis.readByte();
int valueLength = InternalDataSerializer.readArrayLength(dis);
byte[] valueBytes = null;
long drId = DiskInitFile.readDiskRegionID(dis);
DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
// read version
VersionTag tag = null;
if (EntryBits.isWithVersions(userBits)) {
tag = readVersionsFromOplog(dis);
if (drs != null && !drs.getDiskRegionView().getFlags().contains(DiskRegionFlag.IS_WITH_VERSIONING)) {
// 50044 Remove version tag from entry if we don't want versioning
// for this region
tag = null;
userBits = EntryBits.setWithVersions(userBits, false);
} else {
// Update the RVV with the new entry
if (drs != null) {
drs.recordRecoveredVersionTag(tag);
}
}
}
long oplogKeyId = InternalDataSerializer.readVLOld(dis);
long oplogOffset;
if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) {
oplogOffset = -1;
} else {
oplogOffset = lastOffset + InternalDataSerializer.readVLOld(dis);
lastOffset = oplogOffset;
}
if (oplogKeyId > oplogKeyIdHWM) {
oplogKeyIdHWM = oplogKeyId;
}
if (okToSkipModifyRecord(deletedIds, drId, drs, oplogKeyId, true, tag).skip()) {
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry skipping oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
}
this.stats.incRecoveryRecordsSkipped();
incSkipped();
} else {
if (EntryBits.isAnyInvalid(userBits)) {
if (EntryBits.isInvalid(userBits)) {
valueBytes = DiskEntry.INVALID_BYTES;
} else {
valueBytes = DiskEntry.LOCAL_INVALID_BYTES;
}
} else if (EntryBits.isTombstone(userBits)) {
valueBytes = DiskEntry.TOMBSTONE_BYTES;
}
Object key = deserializeKey(keyBytes, version, in);
{
Object oldValue = getRecoveryMap().put(oplogKeyId, key);
if (oldValue != null) {
throw new AssertionError(LocalizedStrings.Oplog_DUPLICATE_CREATE.toLocalizedString(oplogKeyId));
}
}
DiskEntry de = drs.getDiskEntry(key);
if (de == null) {
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
}
DiskEntry.RecoveredEntry re = createRecoveredEntry(valueBytes, valueLength, userBits, getOplogId(), oplogOffset, oplogKeyId, false, version, in);
if (tag != null) {
re.setVersionTag(tag);
}
initRecoveredEntry(drs.getDiskRegionView(), drs.initializeRecoveredEntry(key, re));
drs.getDiskRegionView().incRecoveredEntryCount();
this.stats.incRecoveredEntryCreates();
krfEntryCount++;
} else {
DiskId curdid = de.getDiskId();
// assert curdid.getOplogId() != getOplogId();
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "ignore readNewEntry because getOplogId()={} != curdid.getOplogId()={} for drId={} key={}", getOplogId(), curdid.getOplogId(), drId, key);
}
}
}
keyBytes = DataSerializer.readByteArray(dis);
}
// while
setRecoverNewEntryId(oplogKeyIdHWM);
} catch (IOException ex) {
try {
fis.close();
fis = null;
} catch (IOException ignore) {
}
throw new DiskAccessException("Unable to recover from krf file for oplogId=" + oplogId + ", file=" + f.getName() + ". This file is corrupt, but may be safely deleted.", ex, getParent());
}
if (recoverValues && krfEntryCount > 0) {
oplogsNeedingValueRecovery.add(this);
// TODO optimize this code and make it async
// It should also honor the lru limit
// The fault in logic might not work until
// the region is actually created.
// Instead of reading the crf it might be better to iterate the live
// entry
// list that was built during KRF recovery. Just fault values in until
// we
// hit the LRU limit (if we have one). Only fault in values for entries
// recovered from disk that are still in this oplog.
// Defer faulting in values until all oplogs for the ds have been
// recovered.
}
} finally {
// fix for bug 42776
if (fis != null) {
try {
fis.close();
fis = null;
} catch (IOException ignore) {
}
}
}
return true;
}
use of org.apache.geode.internal.cache.persistence.DiskRecoveryStore in project geode by apache.
the class Oplog method readVersionTagOnlyEntry.
private void readVersionTagOnlyEntry(CountingDataInputStream dis, byte opcode) throws IOException {
long oplogOffset = -1;
long drId = DiskInitFile.readDiskRegionID(dis);
DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
// read versions
VersionTag tag = readVersionsFromOplog(dis);
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readVersionTagOnlyEntry drId={} tag={}", drId, tag);
}
readEndOfRecord(dis);
// Update the RVV with the new entry
if (drs != null) {
drs.recordRecoveredVersionTag(tag);
}
}
use of org.apache.geode.internal.cache.persistence.DiskRecoveryStore in project geode by apache.
the class Oplog method readModifyEntryWithKey.
/**
* Reads an oplog entry of type ModifyWithKey
*
* @param dis DataInputStream from which the oplog is being read
* @param opcode byte whether the id is short/int/long
*/
private void readModifyEntryWithKey(CountingDataInputStream dis, byte opcode, OplogEntryIdSet deletedIds, boolean recoverValue, final LocalRegion currentRegion, Version version, ByteArrayDataInput in, HeapDataOutputStream hdos) throws IOException {
long oplogOffset = -1;
byte userBits = dis.readByte();
int idByteCount = (opcode - OPLOG_MOD_ENTRY_WITH_KEY_1ID) + 1;
// long debugRecoverModEntryId = this.recoverModEntryId;
long oplogKeyId = getModEntryId(dis, idByteCount);
// long debugOplogKeyId = dis.readLong();
// //assert oplogKeyId == debugOplogKeyId
// // : "expected=" + debugOplogKeyId + " actual=" + oplogKeyId
// assert debugRecoverModEntryId == debugOplogKeyId
// : "expected=" + debugOplogKeyId + " actual=" + debugRecoverModEntryId
// + " idByteCount=" + idByteCount
// + " delta=" + this.lastDelta;
long drId = DiskInitFile.readDiskRegionID(dis);
DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
// read version
VersionTag tag = null;
if (EntryBits.isWithVersions(userBits)) {
tag = readVersionsFromOplog(dis);
} else if (getParent().isUpgradeVersionOnly() && drs != null) {
tag = this.createDummyTag(drs);
userBits = EntryBits.setWithVersions(userBits, true);
}
if (drs != null && !drs.getDiskRegionView().getFlags().contains(DiskRegionFlag.IS_WITH_VERSIONING)) {
// 50044 Remove version tag from entry if we don't want versioning for
// this region
tag = null;
userBits = EntryBits.setWithVersions(userBits, false);
}
OkToSkipResult skipResult = okToSkipModifyRecord(deletedIds, drId, drs, oplogKeyId, true, tag);
if (skipResult.skip()) {
if (!isPhase2()) {
incSkipped();
this.stats.incRecoveryRecordsSkipped();
}
} else if (recoverValue && !getParent().isOfflineCompacting()) {
recoverValue = recoverLruValue(drs);
}
byte[] objValue = null;
int valueLength = 0;
CompactionRecord p2cr = null;
long crOffset;
if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) {
if (EntryBits.isInvalid(userBits)) {
objValue = DiskEntry.INVALID_BYTES;
} else if (EntryBits.isTombstone(userBits)) {
objValue = DiskEntry.TOMBSTONE_BYTES;
} else {
objValue = DiskEntry.LOCAL_INVALID_BYTES;
}
crOffset = dis.getCount();
if (!skipResult.skip()) {
if (isPhase2()) {
p2cr = (CompactionRecord) getRecoveryMap().get(oplogKeyId);
if (p2cr != null && p2cr.getOffset() != crOffset) {
skipResult = OkToSkipResult.SKIP_RECORD;
}
}
}
} else {
int len = dis.readInt();
oplogOffset = dis.getCount();
crOffset = oplogOffset;
valueLength = len;
if (!skipResult.skip()) {
if (isPhase2()) {
p2cr = (CompactionRecord) getRecoveryMap().get(oplogKeyId);
if (p2cr != null && p2cr.getOffset() != crOffset) {
skipResult = OkToSkipResult.SKIP_RECORD;
}
}
}
if (!skipResult.skip() && recoverValue) {
byte[] valueBytes = new byte[len];
dis.readFully(valueBytes);
objValue = valueBytes;
validateValue(valueBytes, userBits, version, in);
} else {
forceSkipBytes(dis, len);
}
}
int keyLen = dis.readInt();
incTotalCount();
if (skipResult.skip()) {
if (skipResult.skipKey()) {
forceSkipBytes(dis, keyLen);
} else {
byte[] keyBytes = new byte[keyLen];
dis.readFully(keyBytes);
skippedKeyBytes.put(oplogKeyId, keyBytes);
}
readEndOfRecord(dis);
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "skipping readModEntryWK init oplogKeyId=<{}> drId={}", oplogKeyId, drId);
}
} else {
// read the key
byte[] keyBytes = null;
if (isPhase2()) {
forceSkipBytes(dis, keyLen);
} else {
keyBytes = new byte[keyLen];
dis.readFully(keyBytes);
}
readEndOfRecord(dis);
if (drs != null && tag != null) {
// Update the RVV with the new entry
// This must be done after reading the end of record to make sure
// we don't have a corrupt record. See bug #45538
drs.recordRecoveredVersionTag(tag);
}
assert oplogKeyId >= 0;
if (getParent().isOfflineCompacting()) {
if (isPhase1()) {
CompactionRecord cr = new CompactionRecord(keyBytes, crOffset);
getRecoveryMap().put(oplogKeyId, cr);
drs.getDiskRegionView().incRecoveredEntryCount();
this.stats.incRecoveredEntryCreates();
} else {
// phase2
Assert.assertTrue(p2cr != null, "First pass did not find create a compaction record");
getOplogSet().getChild().copyForwardForOfflineCompact(oplogKeyId, p2cr.getKeyBytes(), objValue, userBits, drId, tag);
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readModifyEntryWithKey copyForward oplogKeyId=<{}>", oplogKeyId);
}
// add it to the deletedIds set so we will ignore it in earlier oplogs
deletedIds.add(oplogKeyId);
}
} else {
Object key = deserializeKey(keyBytes, version, in);
Object oldValue = getRecoveryMap().put(oplogKeyId, key);
if (oldValue != null) {
throw new AssertionError(LocalizedStrings.Oplog_DUPLICATE_CREATE.toLocalizedString(oplogKeyId));
}
// Check the actual region to see if it has this key from
// a previous recovered oplog.
DiskEntry de = drs.getDiskEntry(key);
if (de == null) {
DiskRegionView drv = drs.getDiskRegionView();
// and create an entry
DiskEntry.RecoveredEntry re = createRecoveredEntry(objValue, valueLength, userBits, getOplogId(), oplogOffset, oplogKeyId, recoverValue, version, in);
if (tag != null) {
re.setVersionTag(tag);
}
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readModEntryWK init oplogKeyId=<{}> drId={} key={} oplogOffset={} userBits={} valueLen={} tag={}", oplogKeyId, drId, key, oplogOffset, userBits, valueLength, tag);
}
initRecoveredEntry(drv, drs.initializeRecoveredEntry(key, re));
drs.getDiskRegionView().incRecoveredEntryCount();
this.stats.incRecoveredEntryCreates();
} else {
DiskId curdid = de.getDiskId();
assert curdid.getOplogId() != getOplogId() : "Mutiple ModEntryWK in the same oplog for getOplogId()=" + getOplogId() + " , curdid.getOplogId()=" + curdid.getOplogId() + " , for drId=" + drId + " , key=" + key;
if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
logger.trace(LogMarker.PERSIST_RECOVERY, "ignore readModEntryWK because getOplogId()={} != curdid.getOplogId()={} for drId={} key={}", getOplogId(), curdid.getOplogId(), drId, key);
}
// de = drs.updateRecoveredEntry(key, re);
// updateRecoveredEntry(drv, de, re);
// this.stats.incRecoveredEntryUpdates();
}
}
}
}
use of org.apache.geode.internal.cache.persistence.DiskRecoveryStore in project geode by apache.
the class Oplog method readNewEntry.
/**
* Reads an oplog entry of type Create
*
* @param dis DataInputStream from which the oplog is being read
* @param opcode byte whether the id is short/int/long
*/
private void readNewEntry(CountingDataInputStream dis, byte opcode, OplogEntryIdSet deletedIds, boolean recoverValue, final LocalRegion currentRegion, Version version, ByteArrayDataInput in, HeapDataOutputStream hdos) throws IOException {
final boolean isPersistRecoveryDebugEnabled = logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY);
long oplogOffset = -1;
byte userBits = dis.readByte();
byte[] objValue = null;
int valueLength = 0;
long oplogKeyId = incRecoverNewEntryId();
long drId = DiskInitFile.readDiskRegionID(dis);
DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
// read versions
VersionTag tag = null;
boolean isDummy = false;
if (EntryBits.isWithVersions(userBits)) {
tag = readVersionsFromOplog(dis);
isDummy = false;
} else if (getParent().isUpgradeVersionOnly() && drs != null) {
tag = this.createDummyTag(drs);
userBits = EntryBits.setWithVersions(userBits, true);
isDummy = true;
}
if (drs != null && !drs.getDiskRegionView().getFlags().contains(DiskRegionFlag.IS_WITH_VERSIONING)) {
// 50044 Remove version tag from entry if we don't want versioning for
// this region
tag = null;
userBits = EntryBits.setWithVersions(userBits, false);
}
OkToSkipResult skipResult = okToSkipModifyRecord(deletedIds, drId, drs, oplogKeyId, true, tag);
if (skipResult.skip()) {
if (!isPhase2()) {
this.stats.incRecoveryRecordsSkipped();
incSkipped();
}
} else if (recoverValue && !getParent().isOfflineCompacting()) {
recoverValue = recoverLruValue(drs);
}
CompactionRecord p2cr = null;
long crOffset;
if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) {
if (EntryBits.isInvalid(userBits)) {
objValue = DiskEntry.INVALID_BYTES;
} else if (EntryBits.isTombstone(userBits)) {
objValue = DiskEntry.TOMBSTONE_BYTES;
} else {
objValue = DiskEntry.LOCAL_INVALID_BYTES;
}
crOffset = dis.getCount();
if (!skipResult.skip()) {
if (isPhase2()) {
p2cr = (CompactionRecord) getRecoveryMap().get(oplogKeyId);
if (p2cr != null && p2cr.getOffset() != crOffset) {
skipResult = OkToSkipResult.SKIP_RECORD;
}
}
}
} else {
int len = dis.readInt();
oplogOffset = dis.getCount();
crOffset = oplogOffset;
valueLength = len;
if (!skipResult.skip()) {
if (isPhase2()) {
p2cr = (CompactionRecord) getRecoveryMap().get(oplogKeyId);
if (p2cr != null && p2cr.getOffset() != crOffset) {
skipResult = OkToSkipResult.SKIP_RECORD;
}
}
}
if (recoverValue && !skipResult.skip()) {
byte[] valueBytes = new byte[len];
dis.readFully(valueBytes);
objValue = valueBytes;
validateValue(valueBytes, userBits, version, in);
} else {
forceSkipBytes(dis, len);
}
}
{
int len = dis.readInt();
incTotalCount();
if (skipResult.skip()) {
if (skipResult.skipKey()) {
forceSkipBytes(dis, len);
} else {
byte[] keyBytes = new byte[len];
dis.readFully(keyBytes);
skippedKeyBytes.put(oplogKeyId, keyBytes);
}
readEndOfRecord(dis);
if (drs != null && tag != null) {
// Update the RVV with the new entry
// This must be done after reading the end of record to make sure
// we don't have a corrupt record. See bug #45538
drs.recordRecoveredVersionTag(tag);
}
if (isPersistRecoveryDebugEnabled) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry SKIPPING oplogKeyId=<{}> drId={} userBits={} keyLen={} valueLen={} tag={}", oplogKeyId, drId, userBits, len, valueLength, tag);
}
} else {
byte[] keyBytes = null;
if (isPhase2()) {
forceSkipBytes(dis, len);
} else {
keyBytes = new byte[len];
dis.readFully(keyBytes);
}
readEndOfRecord(dis);
if (drs != null && tag != null) {
// Update the RVV with the new entry
// This must be done after reading the end of record to make sure
// we don't have a corrupt record. See bug #45538
drs.recordRecoveredVersionTag(tag);
}
if (getParent().isOfflineCompacting()) {
if (isPhase1()) {
CompactionRecord cr = new CompactionRecord(keyBytes, crOffset);
getRecoveryMap().put(oplogKeyId, cr);
drs.getDiskRegionView().incRecoveredEntryCount();
this.stats.incRecoveredEntryCreates();
} else {
// phase2
Assert.assertTrue(p2cr != null, "First pass did not find create a compaction record");
getOplogSet().getChild().copyForwardForOfflineCompact(oplogKeyId, p2cr.getKeyBytes(), objValue, userBits, drId, tag);
if (isPersistRecoveryDebugEnabled) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry copyForward oplogKeyId=<{}>", oplogKeyId);
}
// add it to the deletedIds set so we will ignore it in earlier
// oplogs
deletedIds.add(oplogKeyId);
}
} else {
Object key = deserializeKey(keyBytes, version, in);
{
Object oldValue = getRecoveryMap().put(oplogKeyId, key);
if (oldValue != null) {
throw new AssertionError(LocalizedStrings.Oplog_DUPLICATE_CREATE.toLocalizedString(oplogKeyId));
}
}
DiskEntry de = drs.getDiskEntry(key);
if (de == null) {
if (isPersistRecoveryDebugEnabled) {
logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry oplogKeyId=<{}> drId={} key={} userBits={} oplogOffset={} valueLen={} tag={}", oplogKeyId, drId, key, userBits, oplogOffset, valueLength, tag);
}
DiskEntry.RecoveredEntry re = createRecoveredEntry(objValue, valueLength, userBits, getOplogId(), oplogOffset, oplogKeyId, recoverValue, version, in);
if (tag != null) {
re.setVersionTag(tag);
}
initRecoveredEntry(drs.getDiskRegionView(), drs.initializeRecoveredEntry(key, re));
drs.getDiskRegionView().incRecoveredEntryCount();
this.stats.incRecoveredEntryCreates();
} else {
DiskId curdid = de.getDiskId();
assert curdid.getOplogId() != getOplogId();
if (isPersistRecoveryDebugEnabled) {
logger.trace(LogMarker.PERSIST_RECOVERY, "ignore readNewEntry because getOplogId()={} != curdid.getOplogId()={} for drId={} key={}", getOplogId(), curdid.getOplogId(), drId, key);
}
}
}
}
}
}
use of org.apache.geode.internal.cache.persistence.DiskRecoveryStore in project geode by apache.
the class VersionedStatsDiskRegionEntryOffHeapObjectKey method diskInitialize.
// DO NOT modify this class. It was generated from LeafRegionEntry.cpp
private void diskInitialize(RegionEntryContext context, Object value) {
DiskRecoveryStore drs = (DiskRecoveryStore) context;
DiskStoreImpl ds = drs.getDiskStore();
long maxOplogSize = ds.getMaxOplogSize();
// get appropriate instance of DiskId implementation based on maxOplogSize
this.id = DiskId.createDiskId(maxOplogSize, true, /* is persistence */
ds.needsLinkedList());
Helper.initialize(this, drs, value);
}
Aggregations