Examples with DiskAccessException org.apache.geode.cache.DiskAccessException used on opensource projects

Example 26 with DiskAccessException

use of org.apache.geode.cache.DiskAccessException in project geode by apache.

the class Oplog method basicGetForCompactor.

/**
   * Extracts the Value byte array & UserBit from the OpLog and inserts it in the wrapper Object of
   * type BytesAndBitsForCompactor which is passed
   * 
   * @param offsetInOplog The starting position from which to read the data in the opLog
   * @param bitOnly boolean indicating whether the value needs to be extracted along with the
   *        UserBit or not.
   * @param valueLength The length of the byte array which represents the value
   * @param userBits The userBits of the value.
   * @param wrapper Object of type BytesAndBitsForCompactor. The data is set in the wrapper Object.
   *        The wrapper Object also contains the user bit associated with the entry
   * @return true if data is found false if not
   */
private boolean basicGetForCompactor(DiskRegionView dr, long offsetInOplog, boolean bitOnly, int valueLength, byte userBits, BytesAndBitsForCompactor wrapper) {
    if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits) || bitOnly || valueLength == 0) {
        if (EntryBits.isInvalid(userBits)) {
            wrapper.setData(DiskEntry.INVALID_BYTES, userBits, DiskEntry.INVALID_BYTES.length, false);
        } else if (EntryBits.isTombstone(userBits)) {
            wrapper.setData(DiskEntry.TOMBSTONE_BYTES, userBits, DiskEntry.TOMBSTONE_BYTES.length, false);
        } else {
            wrapper.setData(DiskEntry.LOCAL_INVALID_BYTES, userBits, DiskEntry.LOCAL_INVALID_BYTES.length, false);
        }
    } else {
        try {
            synchronized (this.lock) /* crf */
            {
                if (/*
               * !getParent().isSync() since compactor groups writes &&
               */
                (offsetInOplog + valueLength) > this.crf.bytesFlushed && !this.closed) {
                    // fix for bug 41205
                    flushAllNoSync(true);
                }
                if (!reopenFileIfClosed()) {
                    // fix for bug 40648
                    return false;
                }
                final long writePosition = (this.doneAppending) ? this.crf.bytesFlushed : this.crf.raf.getFilePointer();
                if ((offsetInOplog + valueLength) > writePosition) {
                    throw new DiskAccessException(LocalizedStrings.Oplog_TRIED_TO_SEEK_TO_0_BUT_THE_FILE_LENGTH_IS_1_OPLOG_FILE_OBJECT_USED_FOR_READING_2.toLocalizedString(offsetInOplog + valueLength, writePosition, this.crf.raf), dr.getName());
                } else if (offsetInOplog < 0) {
                    throw new DiskAccessException(LocalizedStrings.Oplog_CANNOT_FIND_RECORD_0_WHEN_READING_FROM_1.toLocalizedString(offsetInOplog, this.diskFile.getPath()), dr.getName());
                }
                try {
                    this.crf.raf.seek(offsetInOplog);
                    this.stats.incOplogSeeks();
                    byte[] valueBytes = null;
                    if (wrapper.getBytes().length < valueLength) {
                        valueBytes = new byte[valueLength];
                        this.crf.raf.readFully(valueBytes);
                    } else {
                        valueBytes = wrapper.getBytes();
                        this.crf.raf.readFully(valueBytes, 0, valueLength);
                    }
                    this.stats.incOplogReads();
                    wrapper.setData(valueBytes, userBits, valueLength, true);
                } finally {
                    // disk io
                    if (!this.doneAppending) {
                        this.crf.raf.seek(writePosition);
                        this.stats.incOplogSeeks();
                    }
                // if (this.closed || this.deleted.get()) {
                // throw new DiskAccessException("attempting get on "
                // + (this.deleted.get() ? "destroyed" : "closed")
                // + " oplog #" + getOplogId(), this.owner);
                // }
                }
            }
        } catch (IOException ex) {
            getParent().getCancelCriterion().checkCancelInProgress(ex);
            throw new DiskAccessException(LocalizedStrings.Oplog_FAILED_READING_FROM_0_OPLOG_DETAILS_1_2_3_4_5_6.toLocalizedString(this.diskFile.getPath(), this.oplogId, offsetInOplog, this.crf.currSize, this.crf.bytesFlushed, Boolean.FALSE, Boolean.FALSE), ex, dr.getName());
        } catch (IllegalStateException ex) {
            checkClosed();
            throw ex;
        }
    }
    return true;
}

Example 27 with DiskAccessException

use of org.apache.geode.cache.DiskAccessException in project geode by apache.

the class Oplog method krfClose.

public void krfClose() {
    boolean allClosed = false;
    try {
        if (this.krf.fos != null) {
            DataSerializer.writeByteArray(null, this.krf.dos);
        } else {
            return;
        }
        this.krf.dos.flush();
        this.krf.fos.getChannel().force(true);
        this.krf.dos.close();
        this.krf.dos = null;
        this.krf.bos.close();
        this.krf.bos = null;
        this.krf.fos.close();
        this.krf.fos = null;
        if (this.krf.keyNum == 0) {
            // this is an empty krf file
            this.krf.f.delete();
            assert this.krf.f.exists() == false;
        } else {
            // Mark that this krf is complete.
            getParent().getDiskInitFile().krfCreate(this.oplogId);
            logger.info(LocalizedMessage.create(LocalizedStrings.Oplog_CREATE_0_1_2, new Object[] { toString(), "krf", getParent().getName() }));
        }
        allClosed = true;
    } catch (IOException e) {
        // TODO Auto-generated catch block
        if (getParent().getDiskAccessException() == null) {
            throw new DiskAccessException("Fail to close krf file " + this.krf.f, e, getParent());
        } else {
            logger.info("Fail to close krf file " + this.krf.f + ", but a DiskAccessException happened ealier", getParent().getDiskAccessException());
        }
    } finally {
        if (!allClosed) {
            // IOException happened during close, delete this krf
            closeAndDeleteKrf();
        }
    }
}

Example 28 with DiskAccessException

use of org.apache.geode.cache.DiskAccessException in project geode by apache.

the class Oplog method recoverDrf.

/**
   * Return bytes read.
   */
long recoverDrf(OplogEntryIdSet deletedIds, boolean alreadyRecoveredOnce, boolean latestOplog) {
    File drfFile = this.drf.f;
    if (drfFile == null) {
        this.haveRecoveredDrf = true;
        return 0L;
    }
    lockCompactor();
    try {
        if (this.haveRecoveredDrf && !getHasDeletes())
            // do this while holding lock
            return 0L;
        if (!this.haveRecoveredDrf) {
            this.haveRecoveredDrf = true;
        }
        logger.info(LocalizedMessage.create(LocalizedStrings.DiskRegion_RECOVERING_OPLOG_0_1_2, new Object[] { toString(), drfFile.getAbsolutePath(), getParent().getName() }));
        this.recoverDelEntryId = DiskStoreImpl.INVALID_ID;
        boolean readLastRecord = true;
        CountingDataInputStream dis = null;
        try {
            int recordCount = 0;
            boolean foundDiskStoreRecord = false;
            FileInputStream fis = null;
            try {
                fis = new FileInputStream(drfFile);
                dis = new CountingDataInputStream(new BufferedInputStream(fis, 32 * 1024), drfFile.length());
                boolean endOfLog = false;
                while (!endOfLog) {
                    if (dis.atEndOfFile()) {
                        endOfLog = true;
                        break;
                    }
                    readLastRecord = false;
                    byte opCode = dis.readByte();
                    if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                        logger.trace(LogMarker.PERSIST_RECOVERY, "drf byte={} location={}", opCode, Long.toHexString(dis.getCount()));
                    }
                    switch(opCode) {
                        case OPLOG_EOF_ID:
                            // we are at the end of the oplog. So we need to back up one byte
                            dis.decrementCount();
                            endOfLog = true;
                            break;
                        case OPLOG_DEL_ENTRY_1ID:
                        case OPLOG_DEL_ENTRY_2ID:
                        case OPLOG_DEL_ENTRY_3ID:
                        case OPLOG_DEL_ENTRY_4ID:
                        case OPLOG_DEL_ENTRY_5ID:
                        case OPLOG_DEL_ENTRY_6ID:
                        case OPLOG_DEL_ENTRY_7ID:
                        case OPLOG_DEL_ENTRY_8ID:
                            readDelEntry(dis, opCode, deletedIds, parent);
                            recordCount++;
                            break;
                        case OPLOG_DISK_STORE_ID:
                            readDiskStoreRecord(dis, this.drf.f);
                            foundDiskStoreRecord = true;
                            recordCount++;
                            break;
                        case OPLOG_MAGIC_SEQ_ID:
                            readOplogMagicSeqRecord(dis, this.drf.f, OPLOG_TYPE.DRF);
                            break;
                        case OPLOG_GEMFIRE_VERSION:
                            readGemfireVersionRecord(dis, this.drf.f);
                            recordCount++;
                            break;
                        case OPLOG_RVV:
                            long idx = dis.getCount();
                            readRVVRecord(dis, this.drf.f, true, latestOplog);
                            recordCount++;
                            break;
                        default:
                            throw new DiskAccessException(LocalizedStrings.Oplog_UNKNOWN_OPCODE_0_FOUND_IN_DISK_OPERATION_LOG.toLocalizedString(opCode), getParent());
                    }
                    readLastRecord = true;
                // @todo
                // if (rgn.isDestroyed()) {
                // break;
                // }
                }
            // while
            } finally {
                if (dis != null) {
                    dis.close();
                }
                if (fis != null) {
                    fis.close();
                }
            }
            if (!foundDiskStoreRecord && recordCount > 0) {
                throw new DiskAccessException("The oplog file \"" + this.drf.f + "\" does not belong to the init file \"" + getParent().getInitFile() + "\". Drf did not contain a disk store id.", getParent());
            }
        } catch (EOFException ignore) {
        // ignore since a partial record write can be caused by a crash
        } catch (IOException ex) {
            getParent().getCancelCriterion().checkCancelInProgress(ex);
            throw new DiskAccessException(LocalizedStrings.Oplog_FAILED_READING_FILE_DURING_RECOVERY_FROM_0.toLocalizedString(drfFile.getPath()), ex, getParent());
        } catch (CancelException e) {
            if (logger.isDebugEnabled()) {
                logger.debug("Oplog::readOplog:Error in recovery as Cache was closed", e);
            }
        } catch (RegionDestroyedException e) {
            if (logger.isDebugEnabled()) {
                logger.debug("Oplog::readOplog:Error in recovery as Region was destroyed", e);
            }
        } catch (IllegalStateException e) {
            throw e;
        }
        // Add the Oplog size to the Directory Holder which owns this oplog,
        // so that available space is correctly calculated & stats updated.
        long byteCount = 0;
        if (!readLastRecord) {
            // this means that there was a crash
            // and hence we should not continue to read
            // the next oplog
            this.crashed = true;
            if (dis != null) {
                byteCount = dis.getFileLength();
            }
        } else {
            if (dis != null) {
                byteCount = dis.getCount();
            }
        }
        if (!alreadyRecoveredOnce) {
            setRecoveredDrfSize(byteCount);
            this.dirHolder.incrementTotalOplogSize(byteCount);
        }
        return byteCount;
    } finally {
        unlockCompactor();
    }
}

Example 29 with DiskAccessException

use of org.apache.geode.cache.DiskAccessException in project geode by apache.

the class Oplog method readKrf.

private boolean readKrf(OplogEntryIdSet deletedIds, boolean recoverValues, boolean recoverValuesSync, Set<Oplog> oplogsNeedingValueRecovery, boolean latestOplog) {
    File f = new File(this.diskFile.getPath() + KRF_FILE_EXT);
    if (!f.exists()) {
        return false;
    }
    if (!getParent().getDiskInitFile().hasKrf(this.oplogId)) {
        logger.info(LocalizedMessage.create(LocalizedStrings.Oplog_REMOVING_INCOMPLETE_KRF, new Object[] { f.getName(), this.oplogId, getParent().getName() }));
        f.delete();
    }
    // Set krfCreated to true since we have a krf.
    this.krfCreated.set(true);
    // so that we don't try to recreate the krf.
    if (recoverValuesSync) {
        return false;
    }
    FileInputStream fis;
    try {
        fis = new FileInputStream(f);
    } catch (FileNotFoundException ignore) {
        return false;
    }
    try {
        if (getParent().isOffline() && !getParent().FORCE_KRF_RECOVERY) {
            return false;
        }
        logger.info(LocalizedMessage.create(LocalizedStrings.DiskRegion_RECOVERING_OPLOG_0_1_2, new Object[] { toString(), f.getAbsolutePath(), getParent().getName() }));
        this.recoverNewEntryId = DiskStoreImpl.INVALID_ID;
        this.recoverModEntryId = DiskStoreImpl.INVALID_ID;
        this.recoverModEntryIdHWM = DiskStoreImpl.INVALID_ID;
        long oplogKeyIdHWM = DiskStoreImpl.INVALID_ID;
        int krfEntryCount = 0;
        DataInputStream dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
        final Version version = getProductVersionIfOld();
        final ByteArrayDataInput in = new ByteArrayDataInput();
        try {
            try {
                validateOpcode(dis, OPLOG_MAGIC_SEQ_ID);
                readOplogMagicSeqRecord(dis, f, OPLOG_TYPE.KRF);
                validateOpcode(dis, OPLOG_DISK_STORE_ID);
                readDiskStoreRecord(dis, f);
            } catch (DiskAccessException ignore) {
                // Failed to read the file. There are two possibilities. Either this
                // file is in old format which does not have a magic seq in the
                // beginning or this is not a valid file at all. Try reading it as a
                // file in old format
                fis.close();
                fis = new FileInputStream(f);
                dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
                readDiskStoreRecord(dis, f);
            } catch (IllegalStateException ignore) {
                // Failed to read the file. There are two possibilities. Either this
                // is in new format which has a magic seq in the beginning or this is
                // not a valid file at all
                fis.close();
                fis = new FileInputStream(f);
                dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
                readDiskStoreRecord(dis, f);
            }
            readGemfireVersionRecord(dis, f);
            readTotalCountRecord(dis, f);
            readRVVRecord(dis, f, false, latestOplog);
            long lastOffset = 0;
            byte[] keyBytes = DataSerializer.readByteArray(dis);
            while (keyBytes != null) {
                byte userBits = dis.readByte();
                int valueLength = InternalDataSerializer.readArrayLength(dis);
                byte[] valueBytes = null;
                long drId = DiskInitFile.readDiskRegionID(dis);
                DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
                // read version
                VersionTag tag = null;
                if (EntryBits.isWithVersions(userBits)) {
                    tag = readVersionsFromOplog(dis);
                    if (drs != null && !drs.getDiskRegionView().getFlags().contains(DiskRegionFlag.IS_WITH_VERSIONING)) {
                        // 50044 Remove version tag from entry if we don't want versioning
                        // for this region
                        tag = null;
                        userBits = EntryBits.setWithVersions(userBits, false);
                    } else {
                        // Update the RVV with the new entry
                        if (drs != null) {
                            drs.recordRecoveredVersionTag(tag);
                        }
                    }
                }
                long oplogKeyId = InternalDataSerializer.readVLOld(dis);
                long oplogOffset;
                if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) {
                    oplogOffset = -1;
                } else {
                    oplogOffset = lastOffset + InternalDataSerializer.readVLOld(dis);
                    lastOffset = oplogOffset;
                }
                if (oplogKeyId > oplogKeyIdHWM) {
                    oplogKeyIdHWM = oplogKeyId;
                }
                if (okToSkipModifyRecord(deletedIds, drId, drs, oplogKeyId, true, tag).skip()) {
                    if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                        logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry skipping oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
                    }
                    this.stats.incRecoveryRecordsSkipped();
                    incSkipped();
                } else {
                    if (EntryBits.isAnyInvalid(userBits)) {
                        if (EntryBits.isInvalid(userBits)) {
                            valueBytes = DiskEntry.INVALID_BYTES;
                        } else {
                            valueBytes = DiskEntry.LOCAL_INVALID_BYTES;
                        }
                    } else if (EntryBits.isTombstone(userBits)) {
                        valueBytes = DiskEntry.TOMBSTONE_BYTES;
                    }
                    Object key = deserializeKey(keyBytes, version, in);
                    {
                        Object oldValue = getRecoveryMap().put(oplogKeyId, key);
                        if (oldValue != null) {
                            throw new AssertionError(LocalizedStrings.Oplog_DUPLICATE_CREATE.toLocalizedString(oplogKeyId));
                        }
                    }
                    DiskEntry de = drs.getDiskEntry(key);
                    if (de == null) {
                        if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                            logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
                        }
                        DiskEntry.RecoveredEntry re = createRecoveredEntry(valueBytes, valueLength, userBits, getOplogId(), oplogOffset, oplogKeyId, false, version, in);
                        if (tag != null) {
                            re.setVersionTag(tag);
                        }
                        initRecoveredEntry(drs.getDiskRegionView(), drs.initializeRecoveredEntry(key, re));
                        drs.getDiskRegionView().incRecoveredEntryCount();
                        this.stats.incRecoveredEntryCreates();
                        krfEntryCount++;
                    } else {
                        DiskId curdid = de.getDiskId();
                        // assert curdid.getOplogId() != getOplogId();
                        if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                            logger.trace(LogMarker.PERSIST_RECOVERY, "ignore readNewEntry because getOplogId()={} != curdid.getOplogId()={} for drId={} key={}", getOplogId(), curdid.getOplogId(), drId, key);
                        }
                    }
                }
                keyBytes = DataSerializer.readByteArray(dis);
            }
            // while
            setRecoverNewEntryId(oplogKeyIdHWM);
        } catch (IOException ex) {
            try {
                fis.close();
                fis = null;
            } catch (IOException ignore) {
            }
            throw new DiskAccessException("Unable to recover from krf file for oplogId=" + oplogId + ", file=" + f.getName() + ". This file is corrupt, but may be safely deleted.", ex, getParent());
        }
        if (recoverValues && krfEntryCount > 0) {
            oplogsNeedingValueRecovery.add(this);
        // TODO optimize this code and make it async
        // It should also honor the lru limit
        // The fault in logic might not work until
        // the region is actually created.
        // Instead of reading the crf it might be better to iterate the live
        // entry
        // list that was built during KRF recovery. Just fault values in until
        // we
        // hit the LRU limit (if we have one). Only fault in values for entries
        // recovered from disk that are still in this oplog.
        // Defer faulting in values until all oplogs for the ds have been
        // recovered.
        }
    } finally {
        // fix for bug 42776
        if (fis != null) {
            try {
                fis.close();
                fis = null;
            } catch (IOException ignore) {
            }
        }
    }
    return true;
}

Example 30 with DiskAccessException

use of org.apache.geode.cache.DiskAccessException in project geode by apache.

the class Oplog method readDiskStoreRecord.

/**
   * @throws DiskAccessException if this file does not belong to our parent
   */
private void readDiskStoreRecord(DataInput dis, File f) throws IOException {
    long leastSigBits = dis.readLong();
    long mostSigBits = dis.readLong();
    DiskStoreID readDSID = new DiskStoreID(mostSigBits, leastSigBits);
    if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
        logger.trace(LogMarker.PERSIST_RECOVERY, "diskStoreId={}", readDSID);
    }
    readEndOfRecord(dis);
    DiskStoreID dsid = getParent().getDiskStoreID();
    if (!readDSID.equals(dsid)) {
        throw new DiskAccessException("The oplog file \"" + f + "\" does not belong to the init file \"" + getParent().getInitFile() + "\".", getParent());
    }
}