Examples with StringBinding com.sleepycat.bind.tuple.StringBinding used on opensource projects

Example 1 with StringBinding

use of com.sleepycat.bind.tuple.StringBinding in project heritrix3 by internetarchive.

the class BdbContentDigestHistory method start.

@Override
@SuppressWarnings("rawtypes")
public void start() {
    if (isRunning()) {
        return;
    }
    StoredSortedMap<String, Map> historyMap;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        historyDb = bdb.openDatabase(getHistoryDbName(), historyDbConfig(), true);
        historyMap = new StoredSortedMap<String, Map>(historyDb, new StringBinding(), new SerialBinding<Map>(classCatalog, Map.class), true);
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
    store = historyMap;
}

Example 2 with StringBinding

use of com.sleepycat.bind.tuple.StringBinding in project heritrix3 by internetarchive.

the class PreloadedUriPrecedencePolicy method start.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void start() {
    if (isRunning()) {
        return;
    }
    store = null;
    String dbName = PersistProcessor.URI_HISTORY_DBNAME;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        BdbModule.BdbConfig dbConfig = PersistProcessor.HISTORY_DB_CONFIG;
        historyDb = bdb.openDatabase(dbName, dbConfig, true);
        SerialBinding sb = new SerialBinding(classCatalog, Map.class);
        StoredSortedMap historyMap = new StoredSortedMap(historyDb, new StringBinding(), sb, true);
        store = historyMap;
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
}

Example 3 with StringBinding

use of com.sleepycat.bind.tuple.StringBinding in project heritrix3 by internetarchive.

the class PrefixFinderTest method testStoredSortedMap.

public void testStoredSortedMap() throws Exception {
    EnvironmentConfig config = new EnvironmentConfig();
    config.setAllowCreate(true);
    config.setCachePercent(5);
    File f = new File(getTmpDir(), "PrefixFinderText");
    FileUtils.deleteQuietly(f);
    org.archive.util.FileUtils.ensureWriteableDirectory(f);
    Environment bdbEnvironment = new Environment(f, config);
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setAllowCreate(true);
    dbConfig.setDeferredWrite(true);
    Database db = bdbEnvironment.openDatabase(null, "test", dbConfig);
    StoredSortedMap<String, String> ssm = new StoredSortedMap<String, String>(db, new StringBinding(), new StringBinding(), true);
    testUrlsNoMatch(ssm);
    db.close();
    bdbEnvironment.close();
}

Example 4 with StringBinding

use of com.sleepycat.bind.tuple.StringBinding in project heritrix3 by internetarchive.

the class PrecedenceLoader method main2args.

/**
 * Merge the precalculated precedence information in the first argument
 * file to the environment in the second environment (path; environment
 * will be created if it does not already exist).
 *
 * @param args command-line arguments
 * @throws DatabaseException
 * @throws FileNotFoundException
 * @throws UnsupportedEncodingException
 * @throws IOException
 */
private static void main2args(String[] args) throws DatabaseException, FileNotFoundException, UnsupportedEncodingException, IOException {
    File source = new File(args[0]);
    File env = new File(args[1]);
    FileUtils.ensureWriteableDirectory(env);
    // setup target environment
    EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env);
    StoredClassCatalog classCatalog = targetEnv.getClassCatalog();
    Database historyDB = targetEnv.openDatabase(null, PersistProcessor.URI_HISTORY_DBNAME, PersistProcessor.HISTORY_DB_CONFIG.toDatabaseConfig());
    @SuppressWarnings({ "rawtypes", "unchecked" }) StoredSortedMap<String, Object> historyMap = new StoredSortedMap<String, Object>(historyDB, new StringBinding(), new SerialBinding(classCatalog, Map.class), true);
    int count = 0;
    if (source.isFile()) {
        // scan log, writing to database
        BufferedReader br = ArchiveUtils.getBufferedReader(source);
        Iterator<String> iter = new LineReadingIterator(br);
        while (iter.hasNext()) {
            String line = (String) iter.next();
            String[] splits = line.split("\\s");
            String uri = splits[0];
            if (!uri.matches("\\w+:.*")) {
                // prepend "http://"
                uri = "http://" + uri;
            }
            String key = PersistProcessor.persistKeyFor(uri);
            int precedence = Integer.parseInt(splits[1]);
            @SuppressWarnings("unchecked") Map<String, Object> map = (Map<String, Object>) historyMap.get(key);
            if (map == null) {
                map = new HashMap<String, Object>();
            }
            map.put(A_PRECALC_PRECEDENCE, precedence);
            historyMap.put(key, map);
            count++;
            if (count % 100000 == 0) {
                System.out.print(count + "... ");
            }
        }
        br.close();
        System.out.println();
        System.out.println(count + " entries loaded");
    } else {
        // error
        System.err.println("unacceptable source file");
        return;
    }
    // cleanup
    historyDB.sync();
    historyDB.close();
    targetEnv.close();
    System.out.println(count + " records imported from " + source + " to BDB env " + env);
}

Example 5 with StringBinding

use of com.sleepycat.bind.tuple.StringBinding in project heritrix3 by internetarchive.

the class PersistProcessor method copyPersistEnv.

/**
 * Copies entries from an existing environment db to a new one. If
 * historyMap is not provided, only logs the entries that would have been
 * copied.
 *
 * @param sourceDir existing environment database directory
 * @param historyMap new environment db (or null for a dry run)
 * @return number of records
 * @throws DatabaseException
 */
private static int copyPersistEnv(File sourceDir, StoredSortedMap<String, Map> historyMap) throws DatabaseException {
    int count = 0;
    // open the source env history DB, copying entries to target env
    EnhancedEnvironment sourceEnv = setupCopyEnvironment(sourceDir, true);
    StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog();
    DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig();
    historyDbConfig.setReadOnly(true);
    Database sourceHistoryDB = sourceEnv.openDatabase(null, URI_HISTORY_DBNAME, historyDbConfig);
    StoredSortedMap<String, Map> sourceHistoryMap = new StoredSortedMap<String, Map>(sourceHistoryDB, new StringBinding(), new SerialBinding<Map>(sourceClassCatalog, Map.class), true);
    Iterator<Entry<String, Map>> iter = sourceHistoryMap.entrySet().iterator();
    while (iter.hasNext()) {
        Entry<String, Map> item = iter.next();
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(item.getKey() + " " + new JSONObject(item.getValue()));
        }
        if (historyMap != null) {
            historyMap.put(item.getKey(), item.getValue());
        }
        count++;
    }
    StoredIterator.close(iter);
    sourceHistoryDB.close();
    sourceEnv.close();
    return count;
}