Examples with EnhancedEnvironment org.archive.util.bdbje.EnhancedEnvironment used on opensource projects

Example 1 with EnhancedEnvironment

use of org.archive.util.bdbje.EnhancedEnvironment in project heritrix3 by internetarchive.

the class BdbModule method setup.

protected void setup(File f, boolean create) throws DatabaseException, IOException {
    EnvironmentConfig config = new EnvironmentConfig();
    config.setAllowCreate(create);
    // set to max
    config.setLockTimeout(75, TimeUnit.MINUTES);
    if (getCacheSize() > 0) {
        config.setCacheSize(getCacheSize());
        if (getCachePercent() > 0) {
            LOGGER.warning("cachePercent and cacheSize are both set. Only cacheSize will be used.");
        }
    } else if (getCachePercent() > 0) {
        config.setCachePercent(getCachePercent());
    }
    config.setSharedCache(getUseSharedCache());
    // we take the advice literally from...
    // https://web.archive.org/web/20100727081707/http://www.oracle.com/technology/products/berkeley-db/faq/je_faq.html#33
    long nLockTables = getExpectedConcurrency() - 1;
    while (!BigInteger.valueOf(nLockTables).isProbablePrime(Integer.MAX_VALUE)) {
        nLockTables--;
    }
    config.setConfigParam("je.lock.nLockTables", Long.toString(nLockTables));
    // configure the number of cleaner threads, to speed up clearing out old state files:
    int cleaners = getCleanerThreads();
    if (cleaners > 0) {
        config.setConfigParam(EnvironmentConfig.CLEANER_THREADS, Integer.toString(cleaners));
    }
    // configure number if evictor threads, to avoid critical eviction slowdowns:
    int evictors = this.getEvictorCoreThreads();
    if (evictors > -1) {
        config.setConfigParam(EnvironmentConfig.EVICTOR_CORE_THREADS, Integer.toString(evictors));
    }
    int maxEvictors = this.getEvictorMaxThreads();
    if (maxEvictors > 0) {
        config.setConfigParam(EnvironmentConfig.EVICTOR_MAX_THREADS, Integer.toString(maxEvictors));
    }
    // triple this value to 6K because stats show many faults
    config.setConfigParam("je.log.faultReadSize", "6144");
    // set max bdb log file size. default 10M
    config.setConfigParam("je.log.fileMax", Long.toString(getMaxLogFileSize()));
    if (!getUseHardLinkCheckpoints()) {
        // to support checkpoints by textual manifest only,
        // prevent BDB's cleaner from deleting log files
        config.setConfigParam("je.cleaner.expunge", "false");
    }
    // else leave whatever other setting was already in place
    org.archive.util.FileUtils.ensureWriteableDirectory(f);
    this.bdbEnvironment = new EnhancedEnvironment(f, config);
    this.classCatalog = this.bdbEnvironment.getClassCatalog();
    if (!create) {
        // freeze last log file -- so that originating checkpoint isn't fouled
        DbBackup dbBackup = new DbBackup(bdbEnvironment);
        dbBackup.startBackup();
        dbBackup.endBackup();
    }
}

Example 2 with EnhancedEnvironment

use of org.archive.util.bdbje.EnhancedEnvironment in project heritrix3 by internetarchive.

the class StoredQueueTest method setUp.

protected void setUp() throws Exception {
    super.setUp();
    this.envDir = new File(getTmpDir(), "StoredMapTest");
    org.archive.util.FileUtils.ensureWriteableDirectory(this.envDir);
    try {
        EnvironmentConfig envConfig = new EnvironmentConfig();
        envConfig.setTransactional(false);
        envConfig.setAllowCreate(true);
        env = new EnhancedEnvironment(envDir, envConfig);
        BdbModule.BdbConfig dbConfig = StoredQueue.databaseConfig();
        db = env.openDatabase(null, "StoredMapTest", dbConfig.toDatabaseConfig());
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
    this.queue = new StoredQueue<String>(db, String.class, env.getClassCatalog());
}

Example 3 with EnhancedEnvironment

use of org.archive.util.bdbje.EnhancedEnvironment in project heritrix3 by internetarchive.

the class PrecedenceLoader method main2args.

/**
 * Merge the precalculated precedence information in the first argument
 * file to the environment in the second environment (path; environment
 * will be created if it does not already exist).
 *
 * @param args command-line arguments
 * @throws DatabaseException
 * @throws FileNotFoundException
 * @throws UnsupportedEncodingException
 * @throws IOException
 */
private static void main2args(String[] args) throws DatabaseException, FileNotFoundException, UnsupportedEncodingException, IOException {
    File source = new File(args[0]);
    File env = new File(args[1]);
    FileUtils.ensureWriteableDirectory(env);
    // setup target environment
    EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env);
    StoredClassCatalog classCatalog = targetEnv.getClassCatalog();
    Database historyDB = targetEnv.openDatabase(null, PersistProcessor.URI_HISTORY_DBNAME, PersistProcessor.HISTORY_DB_CONFIG.toDatabaseConfig());
    @SuppressWarnings({ "rawtypes", "unchecked" }) StoredSortedMap<String, Object> historyMap = new StoredSortedMap<String, Object>(historyDB, new StringBinding(), new SerialBinding(classCatalog, Map.class), true);
    int count = 0;
    if (source.isFile()) {
        // scan log, writing to database
        BufferedReader br = ArchiveUtils.getBufferedReader(source);
        Iterator<String> iter = new LineReadingIterator(br);
        while (iter.hasNext()) {
            String line = (String) iter.next();
            String[] splits = line.split("\\s");
            String uri = splits[0];
            if (!uri.matches("\\w+:.*")) {
                // prepend "http://"
                uri = "http://" + uri;
            }
            String key = PersistProcessor.persistKeyFor(uri);
            int precedence = Integer.parseInt(splits[1]);
            @SuppressWarnings("unchecked") Map<String, Object> map = (Map<String, Object>) historyMap.get(key);
            if (map == null) {
                map = new HashMap<String, Object>();
            }
            map.put(A_PRECALC_PRECEDENCE, precedence);
            historyMap.put(key, map);
            count++;
            if (count % 100000 == 0) {
                System.out.print(count + "... ");
            }
        }
        br.close();
        System.out.println();
        System.out.println(count + " entries loaded");
    } else {
        // error
        System.err.println("unacceptable source file");
        return;
    }
    // cleanup
    historyDB.sync();
    historyDB.close();
    targetEnv.close();
    System.out.println(count + " records imported from " + source + " to BDB env " + env);
}

Example 4 with EnhancedEnvironment

use of org.archive.util.bdbje.EnhancedEnvironment in project heritrix3 by internetarchive.

the class PersistProcessor method setupCopyEnvironment.

public static EnhancedEnvironment setupCopyEnvironment(File env, boolean readOnly) throws DatabaseException {
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    envConfig.setReadOnly(readOnly);
    try {
        return new EnhancedEnvironment(env, envConfig);
    } catch (IllegalArgumentException iae) {
        throw new IllegalArgumentException("problem with specified environment " + env + "; is it already open?", iae);
    }
}

Example 5 with EnhancedEnvironment

use of org.archive.util.bdbje.EnhancedEnvironment in project heritrix3 by internetarchive.

the class PersistProcessor method copyPersistEnv.

/**
 * Copies entries from an existing environment db to a new one. If
 * historyMap is not provided, only logs the entries that would have been
 * copied.
 *
 * @param sourceDir existing environment database directory
 * @param historyMap new environment db (or null for a dry run)
 * @return number of records
 * @throws DatabaseException
 */
private static int copyPersistEnv(File sourceDir, StoredSortedMap<String, Map> historyMap) throws DatabaseException {
    int count = 0;
    // open the source env history DB, copying entries to target env
    EnhancedEnvironment sourceEnv = setupCopyEnvironment(sourceDir, true);
    StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog();
    DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig();
    historyDbConfig.setReadOnly(true);
    Database sourceHistoryDB = sourceEnv.openDatabase(null, URI_HISTORY_DBNAME, historyDbConfig);
    StoredSortedMap<String, Map> sourceHistoryMap = new StoredSortedMap<String, Map>(sourceHistoryDB, new StringBinding(), new SerialBinding<Map>(sourceClassCatalog, Map.class), true);
    Iterator<Entry<String, Map>> iter = sourceHistoryMap.entrySet().iterator();
    while (iter.hasNext()) {
        Entry<String, Map> item = iter.next();
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(item.getKey() + " " + new JSONObject(item.getValue()));
        }
        if (historyMap != null) {
            historyMap.put(item.getKey(), item.getValue());
        }
        count++;
    }
    StoredIterator.close(iter);
    sourceHistoryDB.close();
    sourceEnv.close();
    return count;
}