use of org.apache.derby.iapi.services.io.Formatable in project derby by apache.
the class LogToFile method recover.
/**
* Recover the rawStore to a consistent state using the log.
*
* <P>
* In this implementation, the log is a stream of log records stored in
* one or more flat files. Recovery is done in 2 passes: redo and undo.
* <BR> <B>Redo pass</B>
* <BR> In the redo pass, reconstruct the state of the rawstore by
* repeating exactly what happened before as recorded in the log.
* <BR><B>Undo pass</B>
* <BR> In the undo pass, all incomplete transactions are rolled back in
* the order from the most recently started to the oldest.
*
* <P>MT - synchronization provided by caller - RawStore boot.
* This method is guaranteed to be the only method being called and can
* assume single thread access on all fields.
*
* @see Loggable#needsRedo
* @see FileLogger#redo
*
* @exception StandardException Standard Derby error policy
*/
public void recover(DataFactory df, TransactionFactory tf) throws StandardException {
if (SanityManager.DEBUG) {
SanityManager.ASSERT(df != null, "data factory == null");
}
checkCorrupt();
dataFactory = df;
// to encrypt checksum log records.
if (firstLog != null)
logOut = new LogAccessFile(this, firstLog, logBufferSize);
// initialization without causing serialization conflicts.
if (inReplicationSlaveMode) {
synchronized (slaveRecoveryMonitor) {
// while this thread waited on the monitor
while (inReplicationSlaveMode && (allowedToReadFileNumber < bootTimeLogFileNumber)) {
// Wait until the first log file can be read.
if (replicationSlaveException != null) {
throw replicationSlaveException;
}
try {
slaveRecoveryMonitor.wait();
} catch (InterruptedException ie) {
InterruptStatus.setInterrupted();
}
}
}
}
if (recoveryNeeded) {
try {
// ///////////////////////////////////////////////////////////
//
// During boot time, the log control file is accessed and
// bootTimeLogFileNumber is determined. LogOut is not set up.
// bootTimeLogFileNumber is the log file the latest checkpoint
// lives in,
// or 1. It may not be the latest log file (the system may have
// crashed between the time a new log was generated and the
// checkpoint log written), that can only be determined at the
// end of recovery redo.
//
// ///////////////////////////////////////////////////////////
FileLogger logger = (FileLogger) getLogger();
// ///////////////////////////////////////////////////////////
if (checkpointInstant != LogCounter.INVALID_LOG_INSTANT) {
currentCheckpoint = findCheckpoint(checkpointInstant, logger);
}
// beginning of the first log file
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(DUMP_LOG_ONLY)) {
currentCheckpoint = null;
System.out.println("Dump log only");
// unless otherwise specified, 1st log file starts at 1
String beginLogFileNumber = PropertyUtil.getSystemProperty(DUMP_LOG_FROM_LOG_FILE);
if (beginLogFileNumber != null) {
bootTimeLogFileNumber = Long.valueOf(beginLogFileNumber).longValue();
} else {
bootTimeLogFileNumber = 1;
}
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON("setCheckpoint")) {
currentCheckpoint = null;
System.out.println("Set Checkpoint.");
// unless otherwise specified, 1st log file starts at 1
String checkpointStartLogStr = PropertyUtil.getSystemProperty("derby.storage.checkpointStartLog");
String checkpointStartOffsetStr = PropertyUtil.getSystemProperty("derby.storage.checkpointStartOffset");
if ((checkpointStartLogStr != null) && (checkpointStartOffsetStr != null)) {
checkpointInstant = LogCounter.makeLogInstantAsLong(Long.valueOf(checkpointStartLogStr).longValue(), Long.valueOf(checkpointStartOffsetStr).longValue());
} else {
SanityManager.THROWASSERT("must set derby.storage.checkpointStartLog and derby.storage.checkpointStartOffset, if setting setCheckpoint.");
}
currentCheckpoint = findCheckpoint(checkpointInstant, logger);
}
}
long redoLWM = LogCounter.INVALID_LOG_INSTANT;
long undoLWM = LogCounter.INVALID_LOG_INSTANT;
long ttabInstant = LogCounter.INVALID_LOG_INSTANT;
StreamLogScan redoScan = null;
if (currentCheckpoint != null) {
Formatable transactionTable = null;
// RESOLVE: sku
// currentCheckpoint.getTransactionTable();
// need to set the transaction table before the undo
tf.useTransactionTable(transactionTable);
redoLWM = currentCheckpoint.redoLWM();
undoLWM = currentCheckpoint.undoLWM();
if (transactionTable != null)
ttabInstant = checkpointInstant;
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(DBG_FLAG)) {
SanityManager.DEBUG(DBG_FLAG, "Found checkpoint at " + LogCounter.toDebugString(checkpointInstant) + " " + currentCheckpoint.toString());
}
}
firstLogFileNumber = LogCounter.getLogFileNumber(redoLWM);
// figure out where the first interesting log file is.
if (LogCounter.getLogFileNumber(undoLWM) < firstLogFileNumber) {
firstLogFileNumber = LogCounter.getLogFileNumber(undoLWM);
}
// if the checkpoint record doesn't have a transaction
// table, we need to rebuild it by scanning the log from
// the undoLWM. If it does have a transaction table, we
// only need to scan the log from the redoLWM
redoScan = (StreamLogScan) openForwardsScan(undoLWM, (LogInstant) null);
} else {
// no checkpoint
tf.useTransactionTable((Formatable) null);
long start = LogCounter.makeLogInstantAsLong(bootTimeLogFileNumber, LOG_FILE_HEADER_SIZE);
// no checkpoint, start redo from the beginning of the
// file - assume this is the first log file
firstLogFileNumber = bootTimeLogFileNumber;
redoScan = (StreamLogScan) openForwardsScan(start, (LogInstant) null);
}
// open a transaction that is used for redo and rollback
RawTransaction recoveryTransaction = tf.startTransaction(rawStoreFactory, getContextService().getCurrentContextManager(), AccessFactoryGlobals.USER_TRANS_NAME);
// make this transaction aware that it is a recovery transaction
// and don't spew forth post commit work while replaying the log
recoveryTransaction.recoveryTransaction();
// ///////////////////////////////////////////////////////////
//
// Redo loop - in FileLogger
//
// ///////////////////////////////////////////////////////////
//
// set log factory state to inRedo so that if redo caused any
// dirty page to be written from the cache, it won't flush the
// log since the end of the log has not been determined and we
// know the log record that caused the page to change has
// already been written to the log. We need the page write to
// go thru the log factory because if the redo has a problem,
// the log factory is corrupt and the only way we know not to
// write out the page in a checkpoint is if it check with the
// log factory, and that is done via a flush - we use the WAL
// protocol to stop corrupt pages from writing to the disk.
//
inRedo = true;
long logEnd = logger.redo(recoveryTransaction, tf, redoScan, redoLWM, ttabInstant);
inRedo = false;
// Replication slave: When recovery has completed the
// redo pass, the database is no longer in replication
// slave mode and only the recover thread will access
// this object until recover has complete. We
// therefore do not need two versions of the log file
// number anymore. From this point on, logFileNumber
// is used for all references to the current log file
// number; bootTimeLogFileNumber is no longer used.
logFileNumber = bootTimeLogFileNumber;
// the database and prevent anyone from using the log
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DUMP_LOG_ONLY)) {
Monitor.logMessage("_____________________________________________________");
Monitor.logMessage("\n\t\t Log dump finished");
Monitor.logMessage("_____________________________________________________");
// just in case, it has not been set anyway
logOut = null;
return;
}
}
// ///////////////////////////////////////////////////////////
//
// determine where the log ends
//
// ///////////////////////////////////////////////////////////
StorageRandomAccessFile theLog = null;
// some way ...
if (logEnd == LogCounter.INVALID_LOG_INSTANT) {
Monitor.logTextMessage(MessageId.LOG_LOG_NOT_FOUND);
StorageFile logFile = getLogFileName(logFileNumber);
if (privExists(logFile)) {
// otherwise, skip it
if (!privDelete(logFile)) {
logFile = getLogFileName(++logFileNumber);
}
}
IOException accessException = null;
try {
theLog = privRandomAccessFile(logFile, "rw");
} catch (IOException ioe) {
theLog = null;
accessException = ioe;
}
if (theLog == null || !privCanWrite(logFile)) {
if (theLog != null)
theLog.close();
theLog = null;
Monitor.logTextMessage(MessageId.LOG_CHANGED_DB_TO_READ_ONLY);
if (accessException != null)
Monitor.logThrowable(accessException);
ReadOnlyDB = true;
} else {
try {
// no previous log file or previous log position
if (!initLogFile(theLog, logFileNumber, LogCounter.INVALID_LOG_INSTANT)) {
throw markCorrupt(StandardException.newException(SQLState.LOG_SEGMENT_NOT_EXIST, logFile.getPath()));
}
} catch (IOException ioe) {
throw markCorrupt(StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
}
// successfully init'd the log file - set up markers,
// and position at the end of the log.
setEndPosition(theLog.getFilePointer());
lastFlush = endPosition;
// and reopen the file in rwd mode.
if (isWriteSynced) {
// extend the file by wring zeros to it
preAllocateNewLogFile(theLog);
theLog.close();
theLog = openLogFileInWriteMode(logFile);
// postion the log at the current end postion
theLog.seek(endPosition);
}
if (SanityManager.DEBUG) {
SanityManager.ASSERT(endPosition == LOG_FILE_HEADER_SIZE, "empty log file has wrong size");
}
// because we already incrementing the log number
// here, no special log switch required for
// backup recoveries.
logSwitchRequired = false;
}
} else {
// logEnd is the instant of the next log record in the log
// it is used to determine the last known good position of
// the log
logFileNumber = LogCounter.getLogFileNumber(logEnd);
ReadOnlyDB = df.isReadOnly();
StorageFile logFile = getLogFileName(logFileNumber);
if (!ReadOnlyDB) {
// if datafactory doesn't think it is readonly, we can
// do some futher test of our own
IOException accessException = null;
try {
if (isWriteSynced)
theLog = openLogFileInWriteMode(logFile);
else
theLog = privRandomAccessFile(logFile, "rw");
} catch (IOException ioe) {
theLog = null;
accessException = ioe;
}
if (theLog == null || !privCanWrite(logFile)) {
if (theLog != null)
theLog.close();
theLog = null;
Monitor.logTextMessage(MessageId.LOG_CHANGED_DB_TO_READ_ONLY);
if (accessException != null)
Monitor.logThrowable(accessException);
ReadOnlyDB = true;
}
}
if (!ReadOnlyDB) {
setEndPosition(LogCounter.getLogFilePosition(logEnd));
// find out if log had incomplete log records at the end.
if (redoScan.isLogEndFuzzy()) {
theLog.seek(endPosition);
long eof = theLog.length();
Monitor.logTextMessage(MessageId.LOG_INCOMPLETE_LOG_RECORD, logFile, endPosition, eof);
/* Write zeros from incomplete log record to end of file */
long nWrites = (eof - endPosition) / logBufferSize;
int rBytes = (int) ((eof - endPosition) % logBufferSize);
byte[] zeroBuf = new byte[logBufferSize];
// write the zeros to file
while (nWrites-- > 0) theLog.write(zeroBuf);
if (rBytes != 0)
theLog.write(zeroBuf, 0, rBytes);
if (!isWriteSynced)
syncFile(theLog);
}
if (SanityManager.DEBUG) {
if (theLog.length() != endPosition) {
SanityManager.ASSERT(theLog.length() > endPosition, "log end > log file length, bad scan");
}
}
// set the log to the true end position,
// and not the end of the file
lastFlush = endPosition;
theLog.seek(endPosition);
}
}
if (theLog != null) {
if (logOut != null) {
// Close the currently open log file, if there is
// one. DERBY-5937.
logOut.close();
}
logOut = new LogAccessFile(this, theLog, logBufferSize);
}
if (logSwitchRequired)
switchLogFile();
boolean noInFlightTransactions = tf.noActiveUpdateTransaction();
if (ReadOnlyDB) {
// dirty buffer
if (!noInFlightTransactions) {
throw StandardException.newException(SQLState.LOG_READ_ONLY_DB_NEEDS_UNDO);
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "About to call undo(), transaction table =" + tf.getTransactionTable());
}
if (!noInFlightTransactions) {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "In recovery undo, rollback inflight transactions");
}
tf.rollbackAllTransactions(recoveryTransaction, rawStoreFactory);
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "finish recovery undo,");
}
} else {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "No in flight transaction, no recovery undo work");
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "About to call rePrepare(), transaction table =" + tf.getTransactionTable());
}
tf.handlePreparedXacts(rawStoreFactory);
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "Finished rePrepare(), transaction table =" + tf.getTransactionTable());
}
// ///////////////////////////////////////////////////////////
//
// End of recovery.
//
// ///////////////////////////////////////////////////////////
// recovery is finished. Close the transaction
recoveryTransaction.close();
// notify the dataFactory that recovery is completed,
// but before the checkpoint is written.
dataFactory.postRecovery();
// ////////////////////////////////////////////////////////////
// set the transaction factory short id, we have seen all the
// trasactions in the log, and at the minimum, the checkpoint
// transaction will be there. Set the shortId to the next
// value.
// ////////////////////////////////////////////////////////////
tf.resetTranId();
// if can't checkpoint for some reasons, flush log and carry on
if (!ReadOnlyDB) {
boolean needCheckpoint = true;
// rollbacks, then don't checkpoint. Otherwise checkpoint.
if (currentCheckpoint != null && noInFlightTransactions && redoLWM != LogCounter.INVALID_LOG_INSTANT && undoLWM != LogCounter.INVALID_LOG_INSTANT) {
if ((logFileNumber == LogCounter.getLogFileNumber(redoLWM)) && (logFileNumber == LogCounter.getLogFileNumber(undoLWM)) && (endPosition < (LogCounter.getLogFilePosition(redoLWM) + 1000)))
needCheckpoint = false;
}
if (needCheckpoint && !checkpoint(rawStoreFactory, df, tf, false))
flush(logFileNumber, endPosition);
}
logger.close();
recoveryNeeded = false;
} catch (IOException ioe) {
if (SanityManager.DEBUG)
ioe.printStackTrace();
throw markCorrupt(StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
} catch (ClassNotFoundException cnfe) {
throw markCorrupt(StandardException.newException(SQLState.LOG_CORRUPTED, cnfe));
} catch (StandardException se) {
throw markCorrupt(se);
} catch (Throwable th) {
if (SanityManager.DEBUG) {
SanityManager.showTrace(th);
th.printStackTrace();
}
throw markCorrupt(StandardException.newException(SQLState.LOG_RECOVERY_FAILED, th));
}
} else {
tf.useTransactionTable((Formatable) null);
// set the transaction factory short id
tf.resetTranId();
}
// done with recovery
// ///////////////////////////////////////////////////////////
// setup checkpoint daemon and cache cleaner
// ///////////////////////////////////////////////////////////
checkpointDaemon = rawStoreFactory.getDaemon();
if (checkpointDaemon != null) {
myClientNumber = checkpointDaemon.subscribe(this, true);
// use the same daemon for the cache cleaner
dataFactory.setupCacheCleaner(checkpointDaemon);
}
}
use of org.apache.derby.iapi.services.io.Formatable in project derby by apache.
the class LogToFile method checkpointWithTran.
/**
* checkpoint with pre-start transaction
*
* @param rsf The RawStoreFactory to use to do the checkpoint.
* @param df The DataFactory to use to do the checkpoint.
* @param tf The TransactionFactory to use to do the checkpoint.
* @param wait If an existing checkpoint is in progress, then if
* wait=true then this routine will wait for the
* checkpoint to complete and the do another checkpoint
* and wait for it to finish before returning.
*
* @exception StandardException Derby Standard Error Policy
*/
private boolean checkpointWithTran(RawTransaction cptran, RawStoreFactory rsf, DataFactory df, TransactionFactory tf, boolean wait) throws StandardException {
LogInstant redoLWM;
// logout is set
if (logOut == null) {
return false;
}
long approxLogLength;
boolean proceed = true;
do {
synchronized (this) {
if (corrupt != null) {
throw StandardException.newException(SQLState.LOG_STORE_CORRUPT, corrupt);
}
// current end position
approxLogLength = endPosition;
if (!inCheckpoint) {
// no checkpoint in progress, change status to indicate
// this code is doing the checkpoint.
inCheckpoint = true;
// in this routine.
break;
} else {
if (wait) {
while (inCheckpoint) {
try {
wait();
} catch (InterruptedException ie) {
InterruptStatus.setInterrupted();
}
}
} else {
// caller did not want to wait for already executing
// checkpoint to finish. Routine will return false
// upon exiting the loop.
proceed = false;
}
}
// don't return from inside of a sync block
}
} while (proceed);
if (!proceed) {
return false;
}
// needCPtran == true if not supplied with a pre-started transaction
boolean needCPTran = (cptran == null);
if (SanityManager.DEBUG) {
if (logSwitchInterval == 0) {
SanityManager.THROWASSERT("switching log file: Approx log length = " + approxLogLength + " logSwitchInterval = 0");
}
}
try {
if (approxLogLength > logSwitchInterval) {
switchLogFile();
// log switch is occuring in conjuction with the
// checkpoint, set the amount of log written from last
// checkpoint to zero.
logWrittenFromLastCheckPoint = 0;
} else {
// checkpoint is happening without the log switch,
// in the middle of a log file. Amount of log written already for
// the current log file should not be included in caluculation
// of when next check point is due. By assigning the negative
// value of amount of log written for this file. Later it will
// be subtracted when we switch the log file or while
// calculating whether we are due a for checkpoint at flush time.
logWrittenFromLastCheckPoint = -endPosition;
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(TEST_LOG_SWITCH_LOG))
return false;
}
// start a checkpoint transaction
if (needCPTran)
cptran = tf.startInternalTransaction(rsf, getContextService().getCurrentContextManager());
// ///////////////////////////////////////////////////
// gather a snapshot of the various interesting points of the log
// ///////////////////////////////////////////////////
long undoLWM_long;
long redoLWM_long;
synchronized (// we could synchronized on something else, it
this) // doesn't matter as long as logAndDo sync on
// the same thing
{
// The redo LWM is the current log instant. We are going to
// clean the cache shortly, any log record before this point
// will not ever need to be redone.
redoLWM_long = currentInstant();
redoLWM = new LogCounter(redoLWM_long);
// The undo LWM is what we need to rollback all transactions.
// Synchronize this with the starting of a new transaction so
// that the transaction factory can have a consistent view
// See FileLogger.logAndDo
LogCounter undoLWM = (LogCounter) (tf.firstUpdateInstant());
if (undoLWM == null)
// no active transaction
undoLWM_long = redoLWM_long;
else
undoLWM_long = undoLWM.getValueAsLong();
}
// ///////////////////////////////////////////////////
// clean the buffer cache
// ///////////////////////////////////////////////////
df.checkpoint();
// ///////////////////////////////////////////////////
// write out the checkpoint log record
// ///////////////////////////////////////////////////
// send the checkpoint record to the log
Formatable transactionTable = tf.getTransactionTable();
CheckpointOperation nextCheckpoint = new CheckpointOperation(redoLWM_long, undoLWM_long, transactionTable);
cptran.logAndDo(nextCheckpoint);
LogCounter checkpointInstant = (LogCounter) (cptran.getLastLogInstant());
if (checkpointInstant != null) {
// since checkpoint is an internal transaction, I need to
// flush it to make sure it actually goes to the log
flush(checkpointInstant);
} else {
throw StandardException.newException(SQLState.LOG_CANNOT_LOG_CHECKPOINT);
}
cptran.commit();
if (needCPTran) {
// if we started it, we will close it
cptran.close();
cptran = null;
}
if (!writeControlFile(getControlFileName(), checkpointInstant.getValueAsLong())) {
throw StandardException.newException(SQLState.LOG_CONTROL_FILE, getControlFileName());
}
// next checkpoint becomes the current checkpoint
currentCheckpoint = nextCheckpoint;
if (!logArchived()) {
truncateLog(currentCheckpoint);
}
// are needed to recover from the backup checkpoint on restore.
if (!backupInProgress)
df.removeDroppedContainerFileStubs(redoLWM);
} catch (IOException ioe) {
throw markCorrupt(StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
} finally {
synchronized (this) {
inCheckpoint = false;
notifyAll();
}
if (cptran != null && needCPTran) {
try {
cptran.commit();
cptran.close();
} catch (StandardException se) {
throw markCorrupt(StandardException.newException(SQLState.LOG_CORRUPTED, se));
}
}
}
return true;
}
use of org.apache.derby.iapi.services.io.Formatable in project derby by apache.
the class CacheLock method setProperty.
/**
* Sets the Serializable object associated with a property key.
* <p>
* This implementation turns the setProperty into an insert into the
* PropertyConglomerate conglomerate.
* <p>
* See the discussion of getProperty().
* <p>
* The value stored may be a Formatable object or a Serializable object
* whose class name starts with java.*. This stops arbitary objects being
* stored in the database by class name, which will cause problems in
* obfuscated/non-obfuscated systems.
*
* @param tc The transaction to do the Conglomerate work under.
* @param key The key used to lookup this property.
* @param value The value to be associated with this key. If null,
* delete the property from the properties list.
*
* @exception StandardException Standard exception policy.
*/
void setProperty(TransactionController tc, String key, Serializable value, boolean dbOnlyProperty) throws StandardException {
if (SanityManager.DEBUG) {
if (!((value == null) || (value instanceof Formatable))) {
if (!(value.getClass().getName().startsWith("java."))) {
SanityManager.THROWASSERT("Non-formattable, non-java class - " + value.getClass().getName());
}
}
}
lockProperties(tc);
Serializable valueToValidateAndApply = value;
// If we remove a property we validate and apply its default.
if (value == null)
valueToValidateAndApply = getPropertyDefault(tc, key);
Serializable valueToSave = validateApplyAndMap(tc, key, valueToValidateAndApply, dbOnlyProperty);
// a special way.
if (bootPasswordChange(tc, key, value))
return;
else // DEFAULT value returned by validateAndApply.
if (value == null)
saveProperty(tc, key, null);
else
//
// value != null means we simply save the possibly
// mapped value of the property returned by
// validateAndApply.
saveProperty(tc, key, valueToSave);
}
Aggregations