Examples with StoredSortedMap com.sleepycat.collections.StoredSortedMap used on opensource projects

Example 1 with StoredSortedMap

use of com.sleepycat.collections.StoredSortedMap in project heritrix3 by internetarchive.

the class PersistProcessor method populatePersistEnvFromLog.

/**
 * Populates an environment db from a persist log. If historyMap is
 * not provided, only logs the entries that would have been populated.
 *
 * @param persistLogReader
 *            persist log
 * @param historyMap
 *            new environment db (or null for a dry run)
 * @return number of records
 * @throws UnsupportedEncodingException
 * @throws DatabaseException
 */
private static int populatePersistEnvFromLog(BufferedReader persistLogReader, StoredSortedMap<String, Map> historyMap) throws UnsupportedEncodingException, DatabaseException {
    int count = 0;
    Iterator<String> iter = new LineReadingIterator(persistLogReader);
    while (iter.hasNext()) {
        String line = iter.next();
        if (line.length() == 0) {
            continue;
        }
        String[] splits = line.split(" ");
        if (splits.length != 2) {
            logger.severe("bad line has " + splits.length + " fields (should be 2): " + line);
            continue;
        }
        Map alist;
        try {
            alist = (Map) SerializationUtils.deserialize(Base64.decodeBase64(splits[1].getBytes("UTF-8")));
        } catch (Exception e) {
            logger.severe("caught exception " + e + " deserializing line: " + line);
            continue;
        }
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(splits[0] + " " + ArchiveUtils.prettyString(alist));
        }
        if (historyMap != null)
            try {
                historyMap.put(splits[0], alist);
            } catch (Exception e) {
                logger.log(Level.SEVERE, "caught exception after loading " + count + " urls from the persist log (perhaps crawl was stopped by user?)", e);
                IOUtils.closeQuietly(persistLogReader);
                // seems to finish most cleanly when we return rather than throw something
                return count;
            }
        count++;
    }
    IOUtils.closeQuietly(persistLogReader);
    return count;
}

Example 2 with StoredSortedMap

use of com.sleepycat.collections.StoredSortedMap in project heritrix3 by internetarchive.

the class BdbContentDigestHistory method start.

@Override
@SuppressWarnings("rawtypes")
public void start() {
    if (isRunning()) {
        return;
    }
    StoredSortedMap<String, Map> historyMap;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        historyDb = bdb.openDatabase(getHistoryDbName(), historyDbConfig(), true);
        historyMap = new StoredSortedMap<String, Map>(historyDb, new StringBinding(), new SerialBinding<Map>(classCatalog, Map.class), true);
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
    store = historyMap;
}

Example 3 with StoredSortedMap

use of com.sleepycat.collections.StoredSortedMap in project heritrix3 by internetarchive.

the class ObjectIdentityBdbManualCache method createDiskMap.

@SuppressWarnings("unchecked")
protected StoredSortedMap<String, V> createDiskMap(Database database, StoredClassCatalog classCatalog, Class valueClass) {
    EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class);
    EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
    if (valueBinding == null) {
        valueBinding = new KryoBinding<V>(valueClass);
    // new SerialBinding(classCatalog, valueClass);
    // new BenchmarkingBinding<V>(new EntryBinding[] {
    // new KryoBinding<V>(valueClass),
    // new RecyclingSerialBinding<V>(classCatalog, valueClass),
    // }, valueClass);
    }
    return new StoredSortedMap<String, V>(database, keyBinding, valueBinding, true);
}

Example 4 with StoredSortedMap

use of com.sleepycat.collections.StoredSortedMap in project heritrix3 by internetarchive.

the class PreloadedUriPrecedencePolicy method start.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void start() {
    if (isRunning()) {
        return;
    }
    store = null;
    String dbName = PersistProcessor.URI_HISTORY_DBNAME;
    try {
        StoredClassCatalog classCatalog = bdb.getClassCatalog();
        BdbModule.BdbConfig dbConfig = PersistProcessor.HISTORY_DB_CONFIG;
        historyDb = bdb.openDatabase(dbName, dbConfig, true);
        SerialBinding sb = new SerialBinding(classCatalog, Map.class);
        StoredSortedMap historyMap = new StoredSortedMap(historyDb, new StringBinding(), sb, true);
        store = historyMap;
    } catch (DatabaseException e) {
        throw new RuntimeException(e);
    }
}

Example 5 with StoredSortedMap

use of com.sleepycat.collections.StoredSortedMap in project heritrix3 by internetarchive.

the class PrefixFinderTest method testStoredSortedMap.

public void testStoredSortedMap() throws Exception {
    EnvironmentConfig config = new EnvironmentConfig();
    config.setAllowCreate(true);
    config.setCachePercent(5);
    File f = new File(getTmpDir(), "PrefixFinderText");
    FileUtils.deleteQuietly(f);
    org.archive.util.FileUtils.ensureWriteableDirectory(f);
    Environment bdbEnvironment = new Environment(f, config);
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setAllowCreate(true);
    dbConfig.setDeferredWrite(true);
    Database db = bdbEnvironment.openDatabase(null, "test", dbConfig);
    StoredSortedMap<String, String> ssm = new StoredSortedMap<String, String>(db, new StringBinding(), new StringBinding(), true);
    testUrlsNoMatch(ssm);
    db.close();
    bdbEnvironment.close();
}