use of org.apache.derby.io.StorageFile in project derby by apache.
the class LogToFile method deleteLogFileAfterCheckpointLogFile.
/*
* delete the log file after the checkpoint.
*
* <P>MT - synchronization provided by caller - RawStore boot,
* This method is called only if a crash occured while
* re-encrypting the database at boot time.
*/
public void deleteLogFileAfterCheckpointLogFile() throws StandardException {
long logFileNumberAfterCheckpoint = LogCounter.getLogFileNumber(checkpointInstant) + 1;
StorageFile logFileAfterCheckpoint = getLogFileName(logFileNumberAfterCheckpoint);
if (privExists(logFileAfterCheckpoint)) {
// with the new key.
if (!privDelete(logFileAfterCheckpoint)) {
// without deleting the log file encyrpted with new key.
throw StandardException.newException(SQLState.UNABLE_TO_DELETE_FILE, logFileAfterCheckpoint);
}
}
}
use of org.apache.derby.io.StorageFile in project derby by apache.
the class LogToFile method restoreLogs.
/**
* This function restores logs based on the following attributes
* are specified on connection URL:
* Attribute.CREATE_FROM (Create database from backup if it does not exist)
* Attribute.RESTORE_FROM (Delete the whole database if it exists and then
* restore it from backup)
* Attribute.ROLL_FORWARD_RECOVERY_FROM:(Perform Rollforward Recovery;
* except for the log directory everything else is replaced by the copy from
* backup. log files in the backup are copied to the existing online log
* directory.
*
* In case of RESTORE_FROM, the whole database directory
* is removed in Directory.java while restoring service.properties
* so even the log directory is removed.
* In case of CREATE_FROM, log directory will not exist if
* we came so far because it should fail if a database already exists.
* In case ROLL_FORWARD_RECOVERY_FROM log directory should not be removed.
* So only thing that needs to be done here is create a
* a log directory if it does not exists and copy the
* log files(including control files) that exists in the backup from which
* we are are trying to restore the database to the online log directory.
*/
private boolean restoreLogs(Properties properties) throws StandardException {
String backupPath = null;
boolean isCreateFrom = false;
boolean isRestoreFrom = false;
// check if the user requested for restore/recovery/create from backup
backupPath = properties.getProperty(Attribute.CREATE_FROM);
if (backupPath != null) {
isCreateFrom = true;
} else {
backupPath = properties.getProperty(Attribute.RESTORE_FROM);
if (backupPath != null) {
isRestoreFrom = true;
} else {
backupPath = properties.getProperty(Attribute.ROLL_FORWARD_RECOVERY_FROM);
// if the backup is not NULL then it is a rollforward recovery.
}
}
if (backupPath != null) {
if (!isCreateFrom) {
if (logDevice == null) {
/**
* In restoreFrom/rollForwardRecoveryFrom mode when no
* logDevice on URL then the log is restored to the same
* location where the log was when backup was taken.
* In createFrom mode behaviour is same as when create=true,
* i.e unless user specifies the logDevice on URL, log will
* be copied to the database home dir.
* Note: LOG_DEVICE_AT_BACKUP will get set if log is not in
* default location(db home).
*/
logDevice = properties.getProperty(Property.LOG_DEVICE_AT_BACKUP);
}
}
getLogStorageFactory();
StorageFile logDir;
logDir = logStorageFactory.newStorageFile(LogFactory.LOG_DIRECTORY_NAME);
// location than the db home.
if (isRestoreFrom && logDevice != null) {
if (!privRemoveDirectory(logDir)) {
// it may be just a file, try deleting it
if (!privDelete(logDir)) {
throw StandardException.newException(SQLState.UNABLE_TO_REMOVE_DATA_DIRECTORY, getLogDirPath(logDir));
}
}
}
// create the log directory.
if (isCreateFrom || isRestoreFrom) {
createLogDirectory();
}
File backupLogDir = new File(backupPath, LogFactory.LOG_DIRECTORY_NAME);
String[] logfilelist = privList(backupLogDir);
if (logfilelist != null) {
for (int i = 0; i < logfilelist.length; i++) {
File blogFile = new File(backupLogDir, logfilelist[i]);
StorageFile clogFile = logStorageFactory.newStorageFile(logDir, logfilelist[i]);
if (!privCopyFile(blogFile, clogFile)) {
throw StandardException.newException(SQLState.UNABLE_TO_COPY_LOG_FILE, blogFile, clogFile);
}
}
} else {
throw StandardException.newException(SQLState.LOG_DIRECTORY_NOT_FOUND_IN_BACKUP, backupLogDir);
}
// we need to switch the log file after redo while
// doing recovery from backups, otherwise we will
// be replacing updated log after a restore withe
// a log in the backup on next restore.
logSwitchRequired = true;
// log is restored from backup.
return true;
} else {
// log is not restored from backup.
return false;
}
}
use of org.apache.derby.io.StorageFile in project derby by apache.
the class LogToFile method getLogFileAtPosition.
/**
* Get a read-only handle to the log file positioned at the stated position
*
* <P> MT- read only
*
* @return null if file does not exist or of the wrong format
* @exception IOException cannot access the log at the new position.
* @exception StandardException Standard Derby error policy
*/
protected StorageRandomAccessFile getLogFileAtPosition(long logInstant) throws IOException, StandardException {
checkCorrupt();
long filenum = LogCounter.getLogFileNumber(logInstant);
long filepos = LogCounter.getLogFilePosition(logInstant);
StorageFile fileName = getLogFileName(filenum);
if (!privExists(fileName)) {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, fileName.getPath() + " does not exist");
}
return null;
}
StorageRandomAccessFile log = null;
try {
log = privRandomAccessFile(fileName, "r");
// verify that the log file is of the right format
if (!verifyLogFormat(log, filenum)) {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, fileName.getPath() + " format mismatch");
}
log.close();
log = null;
} else {
log.seek(filepos);
}
} catch (IOException ioe) {
try {
if (log != null) {
log.close();
log = null;
}
if (SanityManager.DEBUG) {
SanityManager.THROWASSERT("cannot get to position " + filepos + " for log file " + fileName.getPath(), ioe);
}
} catch (IOException ioe2) {
}
throw ioe;
}
return log;
}
use of org.apache.derby.io.StorageFile in project derby by apache.
the class LogToFile method getLogDirectory.
/*
Return the directory the log should go.
<P> MT- read only
@exception StandardException Derby Standard Error Policy
*/
public StorageFile getLogDirectory() throws StandardException {
StorageFile logDir = null;
logDir = logStorageFactory.newStorageFile(LogFactory.LOG_DIRECTORY_NAME);
if (!privExists(logDir)) {
throw StandardException.newException(SQLState.LOG_SEGMENT_NOT_EXIST, logDir.getPath());
}
return logDir;
}
use of org.apache.derby.io.StorageFile in project derby by apache.
the class LogToFile method recover.
/**
* Recover the rawStore to a consistent state using the log.
*
* <P>
* In this implementation, the log is a stream of log records stored in
* one or more flat files. Recovery is done in 2 passes: redo and undo.
* <BR> <B>Redo pass</B>
* <BR> In the redo pass, reconstruct the state of the rawstore by
* repeating exactly what happened before as recorded in the log.
* <BR><B>Undo pass</B>
* <BR> In the undo pass, all incomplete transactions are rolled back in
* the order from the most recently started to the oldest.
*
* <P>MT - synchronization provided by caller - RawStore boot.
* This method is guaranteed to be the only method being called and can
* assume single thread access on all fields.
*
* @see Loggable#needsRedo
* @see FileLogger#redo
*
* @exception StandardException Standard Derby error policy
*/
public void recover(DataFactory df, TransactionFactory tf) throws StandardException {
if (SanityManager.DEBUG) {
SanityManager.ASSERT(df != null, "data factory == null");
}
checkCorrupt();
dataFactory = df;
// to encrypt checksum log records.
if (firstLog != null)
logOut = new LogAccessFile(this, firstLog, logBufferSize);
// initialization without causing serialization conflicts.
if (inReplicationSlaveMode) {
synchronized (slaveRecoveryMonitor) {
// while this thread waited on the monitor
while (inReplicationSlaveMode && (allowedToReadFileNumber < bootTimeLogFileNumber)) {
// Wait until the first log file can be read.
if (replicationSlaveException != null) {
throw replicationSlaveException;
}
try {
slaveRecoveryMonitor.wait();
} catch (InterruptedException ie) {
InterruptStatus.setInterrupted();
}
}
}
}
if (recoveryNeeded) {
try {
// ///////////////////////////////////////////////////////////
//
// During boot time, the log control file is accessed and
// bootTimeLogFileNumber is determined. LogOut is not set up.
// bootTimeLogFileNumber is the log file the latest checkpoint
// lives in,
// or 1. It may not be the latest log file (the system may have
// crashed between the time a new log was generated and the
// checkpoint log written), that can only be determined at the
// end of recovery redo.
//
// ///////////////////////////////////////////////////////////
FileLogger logger = (FileLogger) getLogger();
// ///////////////////////////////////////////////////////////
if (checkpointInstant != LogCounter.INVALID_LOG_INSTANT) {
currentCheckpoint = findCheckpoint(checkpointInstant, logger);
}
// beginning of the first log file
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(DUMP_LOG_ONLY)) {
currentCheckpoint = null;
System.out.println("Dump log only");
// unless otherwise specified, 1st log file starts at 1
String beginLogFileNumber = PropertyUtil.getSystemProperty(DUMP_LOG_FROM_LOG_FILE);
if (beginLogFileNumber != null) {
bootTimeLogFileNumber = Long.valueOf(beginLogFileNumber).longValue();
} else {
bootTimeLogFileNumber = 1;
}
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON("setCheckpoint")) {
currentCheckpoint = null;
System.out.println("Set Checkpoint.");
// unless otherwise specified, 1st log file starts at 1
String checkpointStartLogStr = PropertyUtil.getSystemProperty("derby.storage.checkpointStartLog");
String checkpointStartOffsetStr = PropertyUtil.getSystemProperty("derby.storage.checkpointStartOffset");
if ((checkpointStartLogStr != null) && (checkpointStartOffsetStr != null)) {
checkpointInstant = LogCounter.makeLogInstantAsLong(Long.valueOf(checkpointStartLogStr).longValue(), Long.valueOf(checkpointStartOffsetStr).longValue());
} else {
SanityManager.THROWASSERT("must set derby.storage.checkpointStartLog and derby.storage.checkpointStartOffset, if setting setCheckpoint.");
}
currentCheckpoint = findCheckpoint(checkpointInstant, logger);
}
}
long redoLWM = LogCounter.INVALID_LOG_INSTANT;
long undoLWM = LogCounter.INVALID_LOG_INSTANT;
long ttabInstant = LogCounter.INVALID_LOG_INSTANT;
StreamLogScan redoScan = null;
if (currentCheckpoint != null) {
Formatable transactionTable = null;
// RESOLVE: sku
// currentCheckpoint.getTransactionTable();
// need to set the transaction table before the undo
tf.useTransactionTable(transactionTable);
redoLWM = currentCheckpoint.redoLWM();
undoLWM = currentCheckpoint.undoLWM();
if (transactionTable != null)
ttabInstant = checkpointInstant;
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(DBG_FLAG)) {
SanityManager.DEBUG(DBG_FLAG, "Found checkpoint at " + LogCounter.toDebugString(checkpointInstant) + " " + currentCheckpoint.toString());
}
}
firstLogFileNumber = LogCounter.getLogFileNumber(redoLWM);
// figure out where the first interesting log file is.
if (LogCounter.getLogFileNumber(undoLWM) < firstLogFileNumber) {
firstLogFileNumber = LogCounter.getLogFileNumber(undoLWM);
}
// if the checkpoint record doesn't have a transaction
// table, we need to rebuild it by scanning the log from
// the undoLWM. If it does have a transaction table, we
// only need to scan the log from the redoLWM
redoScan = (StreamLogScan) openForwardsScan(undoLWM, (LogInstant) null);
} else {
// no checkpoint
tf.useTransactionTable((Formatable) null);
long start = LogCounter.makeLogInstantAsLong(bootTimeLogFileNumber, LOG_FILE_HEADER_SIZE);
// no checkpoint, start redo from the beginning of the
// file - assume this is the first log file
firstLogFileNumber = bootTimeLogFileNumber;
redoScan = (StreamLogScan) openForwardsScan(start, (LogInstant) null);
}
// open a transaction that is used for redo and rollback
RawTransaction recoveryTransaction = tf.startTransaction(rawStoreFactory, getContextService().getCurrentContextManager(), AccessFactoryGlobals.USER_TRANS_NAME);
// make this transaction aware that it is a recovery transaction
// and don't spew forth post commit work while replaying the log
recoveryTransaction.recoveryTransaction();
// ///////////////////////////////////////////////////////////
//
// Redo loop - in FileLogger
//
// ///////////////////////////////////////////////////////////
//
// set log factory state to inRedo so that if redo caused any
// dirty page to be written from the cache, it won't flush the
// log since the end of the log has not been determined and we
// know the log record that caused the page to change has
// already been written to the log. We need the page write to
// go thru the log factory because if the redo has a problem,
// the log factory is corrupt and the only way we know not to
// write out the page in a checkpoint is if it check with the
// log factory, and that is done via a flush - we use the WAL
// protocol to stop corrupt pages from writing to the disk.
//
inRedo = true;
long logEnd = logger.redo(recoveryTransaction, tf, redoScan, redoLWM, ttabInstant);
inRedo = false;
// Replication slave: When recovery has completed the
// redo pass, the database is no longer in replication
// slave mode and only the recover thread will access
// this object until recover has complete. We
// therefore do not need two versions of the log file
// number anymore. From this point on, logFileNumber
// is used for all references to the current log file
// number; bootTimeLogFileNumber is no longer used.
logFileNumber = bootTimeLogFileNumber;
// the database and prevent anyone from using the log
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DUMP_LOG_ONLY)) {
Monitor.logMessage("_____________________________________________________");
Monitor.logMessage("\n\t\t Log dump finished");
Monitor.logMessage("_____________________________________________________");
// just in case, it has not been set anyway
logOut = null;
return;
}
}
// ///////////////////////////////////////////////////////////
//
// determine where the log ends
//
// ///////////////////////////////////////////////////////////
StorageRandomAccessFile theLog = null;
// some way ...
if (logEnd == LogCounter.INVALID_LOG_INSTANT) {
Monitor.logTextMessage(MessageId.LOG_LOG_NOT_FOUND);
StorageFile logFile = getLogFileName(logFileNumber);
if (privExists(logFile)) {
// otherwise, skip it
if (!privDelete(logFile)) {
logFile = getLogFileName(++logFileNumber);
}
}
IOException accessException = null;
try {
theLog = privRandomAccessFile(logFile, "rw");
} catch (IOException ioe) {
theLog = null;
accessException = ioe;
}
if (theLog == null || !privCanWrite(logFile)) {
if (theLog != null)
theLog.close();
theLog = null;
Monitor.logTextMessage(MessageId.LOG_CHANGED_DB_TO_READ_ONLY);
if (accessException != null)
Monitor.logThrowable(accessException);
ReadOnlyDB = true;
} else {
try {
// no previous log file or previous log position
if (!initLogFile(theLog, logFileNumber, LogCounter.INVALID_LOG_INSTANT)) {
throw markCorrupt(StandardException.newException(SQLState.LOG_SEGMENT_NOT_EXIST, logFile.getPath()));
}
} catch (IOException ioe) {
throw markCorrupt(StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
}
// successfully init'd the log file - set up markers,
// and position at the end of the log.
setEndPosition(theLog.getFilePointer());
lastFlush = endPosition;
// and reopen the file in rwd mode.
if (isWriteSynced) {
// extend the file by wring zeros to it
preAllocateNewLogFile(theLog);
theLog.close();
theLog = openLogFileInWriteMode(logFile);
// postion the log at the current end postion
theLog.seek(endPosition);
}
if (SanityManager.DEBUG) {
SanityManager.ASSERT(endPosition == LOG_FILE_HEADER_SIZE, "empty log file has wrong size");
}
// because we already incrementing the log number
// here, no special log switch required for
// backup recoveries.
logSwitchRequired = false;
}
} else {
// logEnd is the instant of the next log record in the log
// it is used to determine the last known good position of
// the log
logFileNumber = LogCounter.getLogFileNumber(logEnd);
ReadOnlyDB = df.isReadOnly();
StorageFile logFile = getLogFileName(logFileNumber);
if (!ReadOnlyDB) {
// if datafactory doesn't think it is readonly, we can
// do some futher test of our own
IOException accessException = null;
try {
if (isWriteSynced)
theLog = openLogFileInWriteMode(logFile);
else
theLog = privRandomAccessFile(logFile, "rw");
} catch (IOException ioe) {
theLog = null;
accessException = ioe;
}
if (theLog == null || !privCanWrite(logFile)) {
if (theLog != null)
theLog.close();
theLog = null;
Monitor.logTextMessage(MessageId.LOG_CHANGED_DB_TO_READ_ONLY);
if (accessException != null)
Monitor.logThrowable(accessException);
ReadOnlyDB = true;
}
}
if (!ReadOnlyDB) {
setEndPosition(LogCounter.getLogFilePosition(logEnd));
// find out if log had incomplete log records at the end.
if (redoScan.isLogEndFuzzy()) {
theLog.seek(endPosition);
long eof = theLog.length();
Monitor.logTextMessage(MessageId.LOG_INCOMPLETE_LOG_RECORD, logFile, endPosition, eof);
/* Write zeros from incomplete log record to end of file */
long nWrites = (eof - endPosition) / logBufferSize;
int rBytes = (int) ((eof - endPosition) % logBufferSize);
byte[] zeroBuf = new byte[logBufferSize];
// write the zeros to file
while (nWrites-- > 0) theLog.write(zeroBuf);
if (rBytes != 0)
theLog.write(zeroBuf, 0, rBytes);
if (!isWriteSynced)
syncFile(theLog);
}
if (SanityManager.DEBUG) {
if (theLog.length() != endPosition) {
SanityManager.ASSERT(theLog.length() > endPosition, "log end > log file length, bad scan");
}
}
// set the log to the true end position,
// and not the end of the file
lastFlush = endPosition;
theLog.seek(endPosition);
}
}
if (theLog != null) {
if (logOut != null) {
// Close the currently open log file, if there is
// one. DERBY-5937.
logOut.close();
}
logOut = new LogAccessFile(this, theLog, logBufferSize);
}
if (logSwitchRequired)
switchLogFile();
boolean noInFlightTransactions = tf.noActiveUpdateTransaction();
if (ReadOnlyDB) {
// dirty buffer
if (!noInFlightTransactions) {
throw StandardException.newException(SQLState.LOG_READ_ONLY_DB_NEEDS_UNDO);
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "About to call undo(), transaction table =" + tf.getTransactionTable());
}
if (!noInFlightTransactions) {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "In recovery undo, rollback inflight transactions");
}
tf.rollbackAllTransactions(recoveryTransaction, rawStoreFactory);
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "finish recovery undo,");
}
} else {
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "No in flight transaction, no recovery undo work");
}
}
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "About to call rePrepare(), transaction table =" + tf.getTransactionTable());
}
tf.handlePreparedXacts(rawStoreFactory);
if (SanityManager.DEBUG) {
if (SanityManager.DEBUG_ON(LogToFile.DBG_FLAG))
SanityManager.DEBUG(LogToFile.DBG_FLAG, "Finished rePrepare(), transaction table =" + tf.getTransactionTable());
}
// ///////////////////////////////////////////////////////////
//
// End of recovery.
//
// ///////////////////////////////////////////////////////////
// recovery is finished. Close the transaction
recoveryTransaction.close();
// notify the dataFactory that recovery is completed,
// but before the checkpoint is written.
dataFactory.postRecovery();
// ////////////////////////////////////////////////////////////
// set the transaction factory short id, we have seen all the
// trasactions in the log, and at the minimum, the checkpoint
// transaction will be there. Set the shortId to the next
// value.
// ////////////////////////////////////////////////////////////
tf.resetTranId();
// if can't checkpoint for some reasons, flush log and carry on
if (!ReadOnlyDB) {
boolean needCheckpoint = true;
// rollbacks, then don't checkpoint. Otherwise checkpoint.
if (currentCheckpoint != null && noInFlightTransactions && redoLWM != LogCounter.INVALID_LOG_INSTANT && undoLWM != LogCounter.INVALID_LOG_INSTANT) {
if ((logFileNumber == LogCounter.getLogFileNumber(redoLWM)) && (logFileNumber == LogCounter.getLogFileNumber(undoLWM)) && (endPosition < (LogCounter.getLogFilePosition(redoLWM) + 1000)))
needCheckpoint = false;
}
if (needCheckpoint && !checkpoint(rawStoreFactory, df, tf, false))
flush(logFileNumber, endPosition);
}
logger.close();
recoveryNeeded = false;
} catch (IOException ioe) {
if (SanityManager.DEBUG)
ioe.printStackTrace();
throw markCorrupt(StandardException.newException(SQLState.LOG_IO_ERROR, ioe));
} catch (ClassNotFoundException cnfe) {
throw markCorrupt(StandardException.newException(SQLState.LOG_CORRUPTED, cnfe));
} catch (StandardException se) {
throw markCorrupt(se);
} catch (Throwable th) {
if (SanityManager.DEBUG) {
SanityManager.showTrace(th);
th.printStackTrace();
}
throw markCorrupt(StandardException.newException(SQLState.LOG_RECOVERY_FAILED, th));
}
} else {
tf.useTransactionTable((Formatable) null);
// set the transaction factory short id
tf.resetTranId();
}
// done with recovery
// ///////////////////////////////////////////////////////////
// setup checkpoint daemon and cache cleaner
// ///////////////////////////////////////////////////////////
checkpointDaemon = rawStoreFactory.getDaemon();
if (checkpointDaemon != null) {
myClientNumber = checkpointDaemon.subscribe(this, true);
// use the same daemon for the cache cleaner
dataFactory.setupCacheCleaner(checkpointDaemon);
}
}
Aggregations