Examples with StorageFile org.apache.derby.io.StorageFile used on opensource projects

Example 46 with StorageFile

use of org.apache.derby.io.StorageFile in project derby by apache.

the class RawStore method applyBulkCryptoOperation.

/*
     * Configure the database for encryption, with the  specified 
     * encryption  properties.
     *
     * Basic idea is to encrypt all the containers with new password/key 
     * specified by the user and keep old versions of the data to 
     * rollback the database to the state before the configuration of database 
     * with new encryption attributes. Users can configure the database with 
     * new encryption  attributes at boot time only; advantage of this approach
     * is that there will not be any concurrency issues to handle because
     * no users will be modifying the data. 
     *
     * First step is to encrypt the existing data with new encryption 
     * attributes  and then update the encryption properties for 
     * the database. Configuring  an un-encrypted database for 
     * encryption problem is a minor variation of  re-encrypting an 
     * encrypted database with new encryption key. The database 
     * reconfiguration with new encryption attributes is done under one
     * transaction, if there is a crash/error before it is committed, 
     * then it  is rolled back and the database will be brought back to the
     * state it was before the encryption.  
     *
     * One trickey case in (re) encrypion of database is 
     * unlike standard protocol transaction  commit means all done, 
     * database (re) encryption process has to perform a checkpoint
     *  with a newly generated key then only database  (re) encrption 
     * is complete, Otherwise the problem  is recovery has to deal 
     * with transaction log that is encrypted with old encryption key and 
     * the new encryption key. This probelm is avoided  writing COMMIT
     * and new  CHECKPOINT log record  to a new log file and encrypt the 
     * with a new key, if there is  crash before checkpoint records 
     * are updated , then on next boot the log file after the checkpoint 
     * is deleted before reovery,  which will be the one that is  
     * written with new encryption key and also contains COMMIT record, 
     * so the COMMIT record is also gone when  log file is deleted. 
     * Recovery will not see the commit , so it will  rollback the (re)
     * encryption and revert all the containers to the 
     * original versions. 
     * 
     * Old container versions are deleted only when the check point 
     * with new encryption key is successful, not on post-commit. 
     *
     * @param properties  properties related to this database.
     * @exception StandardException Standard Derby Error Policy
     */
private void applyBulkCryptoOperation(Properties properties, CipherFactory newCipherFactory) throws StandardException {
    boolean decryptDatabase = (isEncryptedDatabase && isTrue(properties, Attribute.DECRYPT_DATABASE));
    boolean reEncrypt = (isEncryptedDatabase && (isSet(properties, Attribute.NEW_BOOT_PASSWORD) || isSet(properties, Attribute.NEW_CRYPTO_EXTERNAL_KEY)));
    if (SanityManager.DEBUG) {
        SanityManager.ASSERT(decryptDatabase || reEncrypt || (!isEncryptedDatabase && isSet(properties, Attribute.DATA_ENCRYPTION)));
    }
    // Check if the cryptographic operation can be performed.
    cryptoOperationAllowed(reEncrypt, decryptDatabase);
    boolean externalKeyEncryption = isSet(properties, Attribute.CRYPTO_EXTERNAL_KEY);
    // check point the datase, so that encryption does not have
    // to encrypt the existing transactions logs.
    logFactory.checkpoint(this, dataFactory, xactFactory, true);
    // start a transaction that is to be used for encryting the database
    RawTransaction transaction = xactFactory.startTransaction(this, getContextService().getCurrentContextManager(), AccessFactoryGlobals.USER_TRANS_NAME);
    try {
        if (decryptDatabase) {
            dataFactory.decryptAllContainers(transaction);
        } else {
            dataFactory.encryptAllContainers(transaction);
        }
        if (SanityManager.DEBUG) {
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_BEFORE_COMMT, reEncrypt);
        }
        // after setting up a new encryption key
        if (!logFactory.isCheckpointInLastLogFile()) {
            // perfrom a checkpoint, this is a reference checkpoint
            // to find if the re(encryption) is complete.
            logFactory.checkpoint(this, dataFactory, xactFactory, true);
        }
        // database is encrypted.
        if (decryptDatabase) {
            isEncryptedDatabase = false;
            logFactory.setDatabaseEncrypted(false, true);
            dataFactory.setDatabaseEncrypted(false);
        } else {
            // Let the log factory know that database is
            // (re-)encrypted and ask it to flush the log
            // before enabling encryption of the log with
            // the new key.
            logFactory.setDatabaseEncrypted(true, true);
            if (reEncrypt) {
                // Switch the encryption/decryption engine to the new ones.
                decryptionEngine = newDecryptionEngine;
                encryptionEngine = newEncryptionEngine;
                currentCipherFactory = newCipherFactory;
            } else {
                // Mark in the raw store that the database is encrypted.
                isEncryptedDatabase = true;
                dataFactory.setDatabaseEncrypted(true);
            }
        }
        // make the log factory ready to encrypt
        // the transaction log with the new encryption
        // key by switching to a new log file.
        // If re-encryption is aborted for any reason,
        // this new log file will be deleted, during
        // recovery.
        logFactory.startNewLogFile();
        // mark that re-encryption is in progress in the
        // service.properties, so that (re) encryption
        // changes that can not be undone using the transaction
        // log can be un-done before recovery starts.
        // (like the changes to service.properties and
        // any log files the can not be understood by the
        // old encryption key), incase engine crashes
        // after this point.
        // if the crash occurs before this point, recovery
        // will rollback the changes using the transaction
        // log.
        properties.put(RawStoreFactory.DB_ENCRYPTION_STATUS, String.valueOf(RawStoreFactory.DB_ENCRYPTION_IN_PROGRESS));
        if (reEncrypt) {
            if (externalKeyEncryption) {
                // save the current copy of verify key file.
                StorageFile verifyKeyFile = storageFactory.newStorageFile(Attribute.CRYPTO_EXTERNAL_KEY_VERIFY_FILE);
                StorageFile oldVerifyKeyFile = storageFactory.newStorageFile(RawStoreFactory.CRYPTO_OLD_EXTERNAL_KEY_VERIFY_FILE);
                if (!privCopyFile(verifyKeyFile, oldVerifyKeyFile))
                    throw StandardException.newException(SQLState.RAWSTORE_ERROR_COPYING_FILE, verifyKeyFile, oldVerifyKeyFile);
                // update the verify key file with the new key info.
                currentCipherFactory.verifyKey(reEncrypt, storageFactory, properties);
            } else {
                // save the current generated encryption key
                String keyString = properties.getProperty(RawStoreFactory.ENCRYPTED_KEY);
                if (keyString != null)
                    properties.put(RawStoreFactory.OLD_ENCRYPTED_KEY, keyString);
            }
        } else if (decryptDatabase) {
            // We cannot remove the encryption properties here, as we may
            // have to revert back to the encrypted database. Instead we set
            // dataEncryption to false and leave all other encryption
            // attributes unchanged. This requires that Derby doesn't store
            // dataEncryption=false for un-encrypted database, otherwise
            // handleIncompleteDbCryptoOperation will be confused.
            properties.put(Attribute.DATA_ENCRYPTION, "false");
        } else {
            // save the encryption block size;
            properties.put(RawStoreFactory.ENCRYPTION_BLOCKSIZE, String.valueOf(encryptionBlockSize));
        }
        // save the new encryption properties into service.properties
        currentCipherFactory.saveProperties(properties);
        if (SanityManager.DEBUG) {
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_AFTER_SWITCH_TO_NEWKEY, reEncrypt);
        }
        // commit the transaction that is used to
        // (re) encrypt the database. Note that
        // this will be logged with newly generated
        // encryption key in the new log file created
        // above.
        transaction.commit();
        if (SanityManager.DEBUG) {
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_AFTER_COMMT, reEncrypt);
        }
        // force the checkpoint with new encryption key.
        logFactory.checkpoint(this, dataFactory, xactFactory, true);
        if (SanityManager.DEBUG) {
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_AFTER_CHECKPOINT, reEncrypt);
        }
        // once the checkpont makes it to the log, re-encrption
        // is complete. only cleanup is remaining ; update the
        // re-encryption status flag to cleanup.
        properties.put(RawStoreFactory.DB_ENCRYPTION_STATUS, String.valueOf(RawStoreFactory.DB_ENCRYPTION_IN_CLEANUP));
        // database is (re)encrypted successfuly,
        // remove the old version of the container files.
        dataFactory.removeOldVersionOfContainers();
        if (decryptDatabase) {
            // By now we can remove all cryptographic properties.
            removeCryptoProperties(properties);
        } else if (reEncrypt) {
            if (externalKeyEncryption) {
                // remove the saved copy of the verify.key file
                StorageFile oldVerifyKeyFile = storageFactory.newStorageFile(RawStoreFactory.CRYPTO_OLD_EXTERNAL_KEY_VERIFY_FILE);
                if (!privDelete(oldVerifyKeyFile))
                    throw StandardException.newException(SQLState.UNABLE_TO_DELETE_FILE, oldVerifyKeyFile);
            } else {
                // remove the old encryption key property.
                properties.remove(RawStoreFactory.OLD_ENCRYPTED_KEY);
            }
        }
        // (re) encrypion is done,  remove the (re)
        // encryption status property.
        properties.remove(RawStoreFactory.DB_ENCRYPTION_STATUS);
        // close the transaction.
        transaction.close();
    } catch (StandardException se) {
        throw StandardException.newException(SQLState.DATABASE_ENCRYPTION_FAILED, se, se.getMessage());
    } finally {
        // clear the new encryption engines.
        newDecryptionEngine = null;
        newEncryptionEngine = null;
    }
}

Example 47 with StorageFile

use of org.apache.derby.io.StorageFile in project derby by apache.

the class EncryptOrDecryptData method restoreContainer.

/* Restore the contaier to the state it was before
     * it was encrypted with new encryption key. This function is
     * called during undo of the EncryptContainerOperation log record
     * incase of a error/crash before database was successfuly configured with
     * new encryption properties.
     * @param ckey the key of the container that needs to be restored.
     * @exception StandardException Standard Derby error policy
     */
void restoreContainer(ContainerKey containerId) throws StandardException {
    if (!dataFactory.getContainerCache().discard(containerId)) {
        if (SanityManager.DEBUG)
            SanityManager.THROWASSERT("unable to discard  container from cache:" + containerId);
    }
    StorageFile currentFile = dataFactory.getContainerPath(containerId, false);
    StorageFile oldFile = getFile(containerId, true);
    StorageFile newFile = getFile(containerId, false);
    // container with the backup copy.
    if (privExists(oldFile)) {
        if (privExists(currentFile)) {
            // rename the current container file to be the new file.
            if (!privRename(currentFile, newFile)) {
                throw StandardException.newException(SQLState.RAWSTORE_ERROR_RENAMING_FILE, currentFile, newFile);
            }
        }
        if (!privRename(oldFile, currentFile)) {
            throw StandardException.newException(SQLState.RAWSTORE_ERROR_RENAMING_FILE, oldFile, currentFile);
        }
    }
    // if the new copy of the container file exists, remove it.
    if (privExists(newFile)) {
        if (!privDelete(newFile))
            throw StandardException.newException(SQLState.UNABLE_TO_DELETE_FILE, newFile);
    }
}

Example 48 with StorageFile

use of org.apache.derby.io.StorageFile in project derby by apache.

the class RawStore method handleIncompleteDbCryptoOperation.

/**
 * Engine might have crashed during encryption of un-encrypted datbase
 * or while re-encryptin an already encrypted database with a new key
 * after all the containers or (re) encrypted. If crash has occured
 * before all containers are encrypted, recovery wil un-do re-encryption
 * using the transaction log, nothing to be done here.
 *
 * If crash has occured after database encryption status flag
 * (RawStoreFactory.DB_ENCRYPTION_STATUS) is set, this method
 * will do any cleanup necessary for the recovery to correctly
 * perform the rollback if required.
 *
 * @param properties  properties related to this database.
 * @exception StandardException Standard Derby Error Policy
 */
public void handleIncompleteDbCryptoOperation(Properties properties) throws StandardException {
    // find what was the encryption status before database crashed.
    int dbEncryptionStatus = 0;
    String dbEncryptionStatusStr = properties.getProperty(RawStoreFactory.DB_ENCRYPTION_STATUS);
    if (dbEncryptionStatusStr != null)
        dbEncryptionStatus = Integer.parseInt(dbEncryptionStatusStr);
    boolean reEncryption = false;
    boolean decryptionFailed = isSet(properties, Attribute.DATA_ENCRYPTION) && !isTrue(properties, Attribute.DATA_ENCRYPTION);
    // check if engine crashed when (re) encryption was in progress.
    if (dbEncryptionStatus == RawStoreFactory.DB_ENCRYPTION_IN_PROGRESS) {
        if (logFactory.isCheckpointInLastLogFile()) {
            // database (re)encryption was successful, only
            // cleanup is remaining. change the status to cleanup.
            dbEncryptionStatus = RawStoreFactory.DB_ENCRYPTION_IN_CLEANUP;
        } else {
            // crash occured before re-encrytion was completed.
            // update the db re-encryption status and write to
            // the service.properties that re-encryption
            // needs to be undone. The reason this status need
            // to be made persistent, it will help to correctly
            // handle a crash in this routine after the log file
            // encrypted with new key is deleted. If this flag
            // is not set, on next reboot, above check
            // will find checkpoint in the last log file and
            // incorrecly assume (re) encryption is
            // successful.
            dbEncryptionStatus = RawStoreFactory.DB_ENCRYPTION_IN_UNDO;
            properties.put(RawStoreFactory.DB_ENCRYPTION_STATUS, String.valueOf(dbEncryptionStatus));
        }
    }
    if (dbEncryptionStatus == RawStoreFactory.DB_ENCRYPTION_IN_UNDO) {
        // delete the log file after the log file that has the checkpoint ,
        // it has the data encrypted with the new key, including the commit
        // record for the transaction that was used to (re)encrypt
        // the database. By Deleting the log file, we are forcing the
        // recovery to rollback the (re)encryption of the database.
        logFactory.deleteLogFileAfterCheckpointLogFile();
        if (SanityManager.DEBUG) {
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_AFTER_RECOVERY_UNDO_LOGFILE_DELETE, reEncryption);
        }
        // Note : If a crash occurs at this point, then on reboot
        // it will again be in the DB_ENRYPTION_IN__UNDO state,
        // there will not be a file after the checkpoint log file,
        // so no file will be deleted.
        // check if this is a external key encryption and
        // if it replace the current verify key file with
        // the old copy.
        StorageFile verifyKeyFile = storageFactory.newStorageFile(Attribute.CRYPTO_EXTERNAL_KEY_VERIFY_FILE);
        if (privExists(verifyKeyFile)) {
            StorageFile oldVerifyKeyFile = storageFactory.newStorageFile(RawStoreFactory.CRYPTO_OLD_EXTERNAL_KEY_VERIFY_FILE);
            if (privExists(oldVerifyKeyFile)) {
                if (!privCopyFile(oldVerifyKeyFile, verifyKeyFile))
                    throw StandardException.newException(SQLState.RAWSTORE_ERROR_COPYING_FILE, oldVerifyKeyFile, verifyKeyFile);
                // only incase of re-encryption there should
                // be old verify key file.
                reEncryption = true;
            } else if (!decryptionFailed) {
                // remove the verify key file.
                if (!privDelete(verifyKeyFile))
                    throw StandardException.newException(SQLState.UNABLE_TO_DELETE_FILE, verifyKeyFile);
            }
        } else {
            // database enrypted with boot password.
            // replace the current encryption key with the old key
            // in the service.properties file.
            // retreive the old encryption key
            String OldKeyString = properties.getProperty(RawStoreFactory.OLD_ENCRYPTED_KEY);
            if (OldKeyString != null) {
                // set the current encrypted key to the old one.
                properties.put(RawStoreFactory.ENCRYPTED_KEY, OldKeyString);
                // only incase of re-encryption there should
                // be old encryted key .
                reEncryption = true;
            }
        }
        if (SanityManager.DEBUG) {
            SanityManager.ASSERT(!(decryptionFailed && reEncryption));
            crashOnDebugFlag(TEST_REENCRYPT_CRASH_AFTER_RECOVERY_UNDO_REVERTING_KEY, reEncryption);
        }
        if (!decryptionFailed && !reEncryption) {
            // Crash occurred when an un-encrypted database was being
            // encrypted, all encryption properties should be removed from
            // service.properties since we are undoing the attempt.
            removeCryptoProperties(properties);
        }
    }
    if (dbEncryptionStatus == RawStoreFactory.DB_ENCRYPTION_IN_CLEANUP) {
        // remove all the old versions of the  containers.
        dataFactory.removeOldVersionOfContainers();
    }
    if (SanityManager.DEBUG) {
        crashOnDebugFlag(TEST_REENCRYPT_CRASH_BEFORE_RECOVERY_FINAL_CLEANUP, reEncryption);
    }
    // either the (re) encryption was complete ,
    // or undone (except for rollback that needs to be
    // done by the recovery). Remove re-encryption specific
    // flags from the service.properties and old copy
    // of the verify key file.
    // delete the old verify key file , if it exists.
    StorageFile oldVerifyKeyFile = storageFactory.newStorageFile(RawStoreFactory.CRYPTO_OLD_EXTERNAL_KEY_VERIFY_FILE);
    if (privExists(oldVerifyKeyFile)) {
        if (!privDelete(oldVerifyKeyFile))
            throw StandardException.newException(SQLState.UNABLE_TO_DELETE_FILE, oldVerifyKeyFile);
    } else {
        // remove the old encryption key property.
        properties.remove(RawStoreFactory.OLD_ENCRYPTED_KEY);
    }
    // Finalize cleanup for failed decryption attempts.
    if (decryptionFailed) {
        if (dbEncryptionStatus == RawStoreFactory.DB_ENCRYPTION_IN_UNDO) {
            // This action is not idempotent in the sense that once set
            // Derby can't detect that what failed was a decryption attempt
            // if DB_ENCRYPTION_STATUS is kept unchanged. Too reduce the
            // window of opportunity this is done here, but should really
            // be atomic with the removal of DB_ENCRYPTION_STATUS.
            properties.setProperty(Attribute.DATA_ENCRYPTION, "true");
        } else {
            removeCryptoProperties(properties);
        }
    }
    // remove the re-encryptin status flag.
    properties.remove(RawStoreFactory.DB_ENCRYPTION_STATUS);
}

Example 49 with StorageFile

use of org.apache.derby.io.StorageFile in project derby by apache.

the class LuceneQueryVTI method initScan.

// ///////////////////////////////////////////////////////////////////
// 
// MINIONS
// 
// ///////////////////////////////////////////////////////////////////
/**
 * Initialize the metadata and scan
 */
private void initScan() throws SQLException {
    try {
        // read the execution context for this AwareVTI
        VTIContext context = getContext();
        _schema = context.vtiSchema();
        String[] nameParts = LuceneSupport.decodeFunctionName(context.vtiTable());
        _table = nameParts[LuceneSupport.TABLE_PART];
        _column = nameParts[LuceneSupport.COLUMN_PART];
        // divine the column names
        VTITemplate.ColumnDescriptor[] returnColumns = getReturnTableSignature(_connection);
        String[] columnNames = new String[returnColumns.length];
        for (int i = 0; i < returnColumns.length; i++) {
            columnNames[i] = returnColumns[i].columnName;
        }
        setColumnNames(columnNames);
        _scoreColumnID = getColumnCount();
        _docIDColumnID = _scoreColumnID - 1;
        _maxKeyID = _docIDColumnID - 1;
        _minKeyID = 1;
        // make sure the user has SELECT privilege on all relevant columns of the underlying table
        vetPrivileges();
        String delimitedColumnName = LuceneSupport.delimitID(_column);
        DerbyLuceneDir derbyLuceneDir = LuceneSupport.getDerbyLuceneDir(_connection, _schema, _table, delimitedColumnName);
        StorageFile propertiesFile = LuceneSupport.getIndexPropertiesFile(derbyLuceneDir);
        Properties indexProperties = readIndexProperties(propertiesFile);
        String indexDescriptorMaker = indexProperties.getProperty(LuceneSupport.INDEX_DESCRIPTOR_MAKER);
        LuceneIndexDescriptor indexDescriptor = getIndexDescriptor(indexDescriptorMaker);
        Analyzer analyzer = indexDescriptor.getAnalyzer();
        QueryParser qp = indexDescriptor.getQueryParser();
        vetLuceneVersion(indexProperties.getProperty(LuceneSupport.LUCENE_VERSION));
        _indexReader = getIndexReader(derbyLuceneDir);
        _searcher = new IndexSearcher(_indexReader);
        Query luceneQuery = qp.parse(_queryText);
        TopScoreDocCollector tsdc = TopScoreDocCollector.create(_windowSize, true);
        if (_scoreCeiling != null) {
            tsdc = TopScoreDocCollector.create(_windowSize, new ScoreDoc(0, _scoreCeiling), true);
        }
        searchAndScore(luceneQuery, tsdc);
    } catch (IOException ioe) {
        throw ToolUtilities.wrap(ioe);
    } catch (ParseException pe) {
        throw ToolUtilities.wrap(pe);
    } catch (PrivilegedActionException pae) {
        throw ToolUtilities.wrap(pae);
    }
}

Example 50 with StorageFile

use of org.apache.derby.io.StorageFile in project derby by apache.

the class LuceneListIndexesVTI method getRowProperties.

/**
 * get the properties of the current row
 */
private Properties getRowProperties() throws SQLException {
    if (rowProperties == null) {
        try {
            readSchemaTableColumn();
            String delimitedColumnName = LuceneSupport.delimitID(column);
            StorageFile indexPropertiesFile = LuceneSupport.getIndexPropertiesFile(connection, schema, table, delimitedColumnName);
            rowProperties = readIndexProperties(indexPropertiesFile);
        } catch (IOException ioe) {
            throw ToolUtilities.wrap(ioe);
        } catch (PrivilegedActionException pae) {
            throw ToolUtilities.wrap(pae);
        }
    }
    return rowProperties;
}