Examples with DbBackup com.sleepycat.je.util.DbBackup used on opensource projects

Example 1 with DbBackup

use of com.sleepycat.je.util.DbBackup in project heritrix3 by internetarchive.

the class BdbModule method doCheckpoint.

public void doCheckpoint(final Checkpoint checkpointInProgress) throws IOException {
    // First sync objectCaches
    for (@SuppressWarnings("rawtypes") ObjectIdentityCache oic : oiCaches.values()) {
        oic.sync();
    }
    try {
        // sync all databases
        for (DatabasePlusConfig dbc : databases.values()) {
            dbc.database.sync();
        }
        // Do a force checkpoint.  That's what a sync does (i.e. doSync).
        CheckpointConfig chkptConfig = new CheckpointConfig();
        chkptConfig.setForce(true);
        // Mark Hayes of sleepycat says:
        // "The default for this property is false, which gives the current
        // behavior (allow deltas).  If this property is true, deltas are
        // prohibited -- full versions of internal nodes are always logged
        // during the checkpoint. When a full version of an internal node
        // is logged during a checkpoint, recovery does not need to process
        // it at all.  It is only fetched if needed by the application,
        // during normal DB operations after recovery. When a delta of an
        // internal node is logged during a checkpoint, recovery must
        // process it by fetching the full version of the node from earlier
        // in the log, and then applying the delta to it.  This can be
        // pretty slow, since it is potentially a large amount of
        // random I/O."
        // chkptConfig.setMinimizeRecoveryTime(true);
        bdbEnvironment.checkpoint(chkptConfig);
        LOGGER.fine("Finished bdb checkpoint.");
        DbBackup dbBackup = new DbBackup(bdbEnvironment);
        try {
            dbBackup.startBackup();
            File envCpDir = new File(dir.getFile(), checkpointInProgress.getName());
            org.archive.util.FileUtils.ensureWriteableDirectory(envCpDir);
            File logfilesList = new File(envCpDir, "jdbfiles.manifest");
            String[] filedata = dbBackup.getLogFilesInBackupSet();
            for (int i = 0; i < filedata.length; i++) {
                File f = new File(dir.getFile(), filedata[i]);
                filedata[i] += "," + f.length();
                if (getUseHardLinkCheckpoints()) {
                    File hardLink = new File(envCpDir, filedata[i]);
                    try {
                        Files.createLink(hardLink.toPath(), f.toPath().toAbsolutePath());
                    } catch (IOException | UnsupportedOperationException e) {
                        LOGGER.log(Level.SEVERE, "unable to create required checkpoint link " + hardLink, e);
                    }
                }
            }
            FileUtils.writeLines(logfilesList, Arrays.asList(filedata));
            LOGGER.fine("Finished processing bdb log files.");
        } finally {
            dbBackup.endBackup();
        }
    } catch (DatabaseException e) {
        throw new IOException(e);
    }
    if (checkpointInProgress.getForgetAllButLatest()) {
        File[] oldEnvCpDirs = dir.getFile().listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                return !name.equals(checkpointInProgress.getName()) && TextUtils.matches("cp\\d{5}-\\d{14}", name);
            }
        });
        for (File d : oldEnvCpDirs) {
            FileUtils.deleteDirectory(d);
        }
    }
}

Example 2 with DbBackup

use of com.sleepycat.je.util.DbBackup in project heritrix3 by internetarchive.

the class BdbModule method setup.

protected void setup(File f, boolean create) throws DatabaseException, IOException {
    EnvironmentConfig config = new EnvironmentConfig();
    config.setAllowCreate(create);
    // set to max
    config.setLockTimeout(75, TimeUnit.MINUTES);
    if (getCacheSize() > 0) {
        config.setCacheSize(getCacheSize());
        if (getCachePercent() > 0) {
            LOGGER.warning("cachePercent and cacheSize are both set. Only cacheSize will be used.");
        }
    } else if (getCachePercent() > 0) {
        config.setCachePercent(getCachePercent());
    }
    config.setSharedCache(getUseSharedCache());
    // we take the advice literally from...
    // https://web.archive.org/web/20100727081707/http://www.oracle.com/technology/products/berkeley-db/faq/je_faq.html#33
    long nLockTables = getExpectedConcurrency() - 1;
    while (!BigInteger.valueOf(nLockTables).isProbablePrime(Integer.MAX_VALUE)) {
        nLockTables--;
    }
    config.setConfigParam("je.lock.nLockTables", Long.toString(nLockTables));
    // configure the number of cleaner threads, to speed up clearing out old state files:
    int cleaners = getCleanerThreads();
    if (cleaners > 0) {
        config.setConfigParam(EnvironmentConfig.CLEANER_THREADS, Integer.toString(cleaners));
    }
    // configure number if evictor threads, to avoid critical eviction slowdowns:
    int evictors = this.getEvictorCoreThreads();
    if (evictors > -1) {
        config.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Integer.toString(evictors));
    }
    int maxEvictors = this.getEvictorMaxThreads();
    if (maxEvictors > 0) {
        config.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Integer.toString(maxEvictors));
    }
    // triple this value to 6K because stats show many faults
    config.setConfigParam("je.log.faultReadSize", "6144");
    // set max bdb log file size. default 10M
    config.setConfigParam("je.log.fileMax", Long.toString(getMaxLogFileSize()));
    if (!getUseHardLinkCheckpoints()) {
        // to support checkpoints by textual manifest only,
        // prevent BDB's cleaner from deleting log files
        config.setConfigParam("je.cleaner.expunge", "false");
    }
    // else leave whatever other setting was already in place
    org.archive.util.FileUtils.ensureWriteableDirectory(f);
    this.bdbEnvironment = new EnhancedEnvironment(f, config);
    this.classCatalog = this.bdbEnvironment.getClassCatalog();
    if (!create) {
        // freeze last log file -- so that originating checkpoint isn't fouled
        DbBackup dbBackup = new DbBackup(bdbEnvironment);
        dbBackup.startBackup();
        dbBackup.endBackup();
    }
}

Example 3 with DbBackup

use of com.sleepycat.je.util.DbBackup in project timbuctoo by HuygensING.

the class BdbBackupper method backupDatabase.

public void backupDatabase(Environment environment, Path dbPath, Path backupPath) throws IOException {
    final Path lastBackuppedFilePath = backupPath.resolve("lastFile");
    final DbBackup backupHelper = lastBackuppedFilePath.toFile().exists() ? new DbBackup(environment, lastFile(lastBackuppedFilePath)) : new DbBackup(environment);
    environment.sync();
    try {
        backupHelper.startBackup();
        final String[] logFilesInBackupSet = backupHelper.getLogFilesInBackupSet();
        for (String logFile : logFilesInBackupSet) {
            Files.copy(dbPath.resolve(logFile), backupPath.resolve(logFile), REPLACE_EXISTING);
        }
        final long lastFileInBackupSet = backupHelper.getLastFileInBackupSet();
        Files.write(lastBackuppedFilePath, Lists.newArrayList("" + lastFileInBackupSet), UTF_8);
    } finally {
        backupHelper.endBackup();
    }
}

Example 4 with DbBackup

use of com.sleepycat.je.util.DbBackup in project voldemort by voldemort.

the class BdbNativeBackup method performBackup.

public void performBackup(File backupDir, AsyncOperationStatus status) {
    // Find the file number of the last file in the previous backup
    // persistently, by either checking the backup archive, or saving
    // state in a persistent file.
    Long lastFileInPrevBackup = determineLastFile(backupDir, status);
    try {
        if (lastFileInPrevBackup == null) {
            backupHelper = new DbBackup(env);
        } else {
            backupHelper = new DbBackup(env, lastFileInPrevBackup);
        }
        // Start backup, find out what needs to be copied.
        System.out.println("Native backup started at " + new Date().toString());
        backupHelper.startBackup();
        try {
            String[] filesForBackup = backupHelper.getLogFilesInBackupSet();
            // Copy the files to archival storage.
            backupFiles(filesForBackup, backupDir, status);
        } finally {
            // Remember to exit backup mode, or all log files won't be
            // cleaned and disk usage will bloat.
            backupHelper.endBackup();
            System.out.println("Native backup completed at " + new Date().toString());
        }
    } catch (Exception e) {
        throw new VoldemortException("Error performing native backup", e);
    }
}

Example 5 with DbBackup

use of com.sleepycat.je.util.DbBackup in project qpid-broker-j by apache.

the class BDBBackup method takeBackup.

/**
 * Creates a backup of the BDB log files in the source directory, copying them to the destination directory.
 *
 * @param fromdir     The source directory path.
 * @param todir       The destination directory path.
 * @param environment An open BDB environment to perform the back up.
 *
 * @throws DatabaseException Any underlying execeptions from BDB are allowed to fall through.
 */
public void takeBackup(String fromdir, String todir, Environment environment) throws DatabaseException {
    DbBackup backupHelper = null;
    try {
        backupHelper = new DbBackup(environment);
        // Prevent BDB from writing to its log files while the backup it taken.
        backupHelper.startBackup();
        // Back up the BDB log files to the destination directory.
        String[] filesForBackup = backupHelper.getLogFilesInBackupSet();
        for (int i = 0; i < filesForBackup.length; i++) {
            File sourceFile = new File(fromdir + File.separator + filesForBackup[i]);
            File destFile = new File(todir + File.separator + filesForBackup[i]);
            FileUtils.copy(sourceFile, destFile);
        }
    } finally {
        // Remember to exit backup mode, or all log files won't be cleaned and disk usage will bloat.
        if (backupHelper != null) {
            backupHelper.endBackup();
        }
    }
}