Examples with SerialBinding com.sleepycat.bind.serial.SerialBinding used on opensource projects

Example 1 with SerialBinding

use of com.sleepycat.bind.serial.SerialBinding in project heritrix3 by internetarchive.

the class BdbContentDigestHistory method start.

@Override
@SuppressWarnings("rawtypes")
public void start() {
    if (isRunning()) {
        return;
    }
    StoredSortedMap<String, Map> historyMap;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        historyDb = bdb.openDatabase(getHistoryDbName(), historyDbConfig(), true);
        historyMap = new StoredSortedMap<String, Map>(historyDb, new StringBinding(), new SerialBinding<Map>(classCatalog, Map.class), true);
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
    store = historyMap;
}

Example 2 with SerialBinding

use of com.sleepycat.bind.serial.SerialBinding in project heritrix3 by internetarchive.

the class PreloadedUriPrecedencePolicy method start.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void start() {
    if (isRunning()) {
        return;
    }
    store = null;
    String dbName = PersistProcessor.URI_HISTORY_DBNAME;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        BdbModule.BdbConfig dbConfig = PersistProcessor.HISTORY_DB_CONFIG;
        historyDb = bdb.openDatabase(dbName, dbConfig, true);
        SerialBinding sb = new SerialBinding(classCatalog, Map.class);
        StoredSortedMap historyMap = new StoredSortedMap(historyDb, new StringBinding(), sb, true);
        store = historyMap;
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
}

Example 3 with SerialBinding

use of com.sleepycat.bind.serial.SerialBinding in project parliament by SemWebCentral.

the class SpatialIndex method createMaps.

private void createMaps() {
    EntryBinding<NodeKey> nodeKeyBinding = new SerialBinding<>(catalog, NodeKey.class);
    EntryBinding<NodeData> nodeDataBinding = new SerialBinding<>(catalog, NodeData.class);
    EntryBinding<Integer> nodeBinding = new SerialBinding<>(catalog, Integer.class);
    nodes = new StoredSortedMap<>(db, nodeKeyBinding, nodeDataBinding, true);
    idNodes = new StoredSortedMap<>(nodeDb, nodeBinding, nodeDataBinding, true);
    Integer maxInt = idNodes.lastKey();
    if (null == maxInt) {
        maxInt = 0;
    } else {
        maxInt = maxInt + 1;
    }
    idCounter.set(maxInt);
    size = db.count();
}

Example 4 with SerialBinding

use of com.sleepycat.bind.serial.SerialBinding in project heritrix3 by internetarchive.

the class PrecedenceLoader method main2args.

/**
 * Merge the precalculated precedence information in the first argument
 * file to the environment in the second environment (path; environment
 * will be created if it does not already exist).
 *
 * @param args command-line arguments
 * @throws DatabaseException
 * @throws FileNotFoundException
 * @throws UnsupportedEncodingException
 * @throws IOException
 */
private static void main2args(String[] args) throws DatabaseException, FileNotFoundException, UnsupportedEncodingException, IOException {
    File source = new File(args[0]);
    File env = new File(args[1]);
    FileUtils.ensureWriteableDirectory(env);
    // setup target environment
    EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env);
    StoredClassCatalog classCatalog = targetEnv.getClassCatalog();
    Database historyDB = targetEnv.openDatabase(null, PersistProcessor.URI_HISTORY_DBNAME, PersistProcessor.HISTORY_DB_CONFIG.toDatabaseConfig());
    @SuppressWarnings({ "rawtypes", "unchecked" }) StoredSortedMap<String, Object> historyMap = new StoredSortedMap<String, Object>(historyDB, new StringBinding(), new SerialBinding(classCatalog, Map.class), true);
    int count = 0;
    if (source.isFile()) {
        // scan log, writing to database
        BufferedReader br = ArchiveUtils.getBufferedReader(source);
        Iterator<String> iter = new LineReadingIterator(br);
        while (iter.hasNext()) {
            String line = (String) iter.next();
            String[] splits = line.split("\\s");
            String uri = splits[0];
            if (!uri.matches("\\w+:.*")) {
                // prepend "http://"
                uri = "http://" + uri;
            }
            String key = PersistProcessor.persistKeyFor(uri);
            int precedence = Integer.parseInt(splits[1]);
            @SuppressWarnings("unchecked") Map<String, Object> map = (Map<String, Object>) historyMap.get(key);
            if (map == null) {
                map = new HashMap<String, Object>();
            }
            map.put(A_PRECALC_PRECEDENCE, precedence);
            historyMap.put(key, map);
            count++;
            if (count % 100000 == 0) {
                System.out.print(count + "... ");
            }
        }
        br.close();
        System.out.println();
        System.out.println(count + " entries loaded");
    } else {
        // error
        System.err.println("unacceptable source file");
        return;
    }
    // cleanup
    historyDB.sync();
    historyDB.close();
    targetEnv.close();
    System.out.println(count + " records imported from " + source + " to BDB env " + env);
}

Example 5 with SerialBinding

use of com.sleepycat.bind.serial.SerialBinding in project heritrix3 by internetarchive.

the class PersistProcessor method populatePersistEnv.

/**
 * Populates a new environment db from an old environment db or a persist
 * log. If path to new environment is not provided, only logs the entries
 * that would have been populated.
 *
 * @param sourcePath
 *            source of old entries: can be a path to an existing
 *            environment db, or a URL or path to a persist log
 * @param envFile
 *            path to new environment db (or null for a dry run)
 * @return number of records
 * @throws DatabaseException
 * @throws IOException
 */
public static int populatePersistEnv(String sourcePath, File envFile) throws IOException {
    int count = 0;
    StoredSortedMap<String, Map> historyMap = null;
    EnhancedEnvironment targetEnv = null;
    StoredClassCatalog classCatalog = null;
    Database historyDB = null;
    if (envFile != null) {
        // set up target environment
        FileUtils.ensureWriteableDirectory(envFile);
        targetEnv = setupCopyEnvironment(envFile);
        classCatalog = targetEnv.getClassCatalog();
        historyDB = targetEnv.openDatabase(null, URI_HISTORY_DBNAME, HISTORY_DB_CONFIG.toDatabaseConfig());
        historyMap = new StoredSortedMap<String, Map>(historyDB, new StringBinding(), new SerialBinding<Map>(classCatalog, Map.class), true);
    }
    try {
        count = copyPersistSourceToHistoryMap(new File(sourcePath), historyMap);
    } finally {
        // failed to populate it
        if (envFile != null) {
            logger.info(count + " records imported from " + sourcePath + " to BDB env " + envFile);
            historyDB.sync();
            historyDB.close();
            targetEnv.close();
        } else {
            logger.info(count + " records found in " + sourcePath);
        }
    }
    return count;
}