Examples with BdbModule org.archive.bdb.BdbModule used on opensource projects

Example 1 with BdbModule

use of org.archive.bdb.BdbModule in project heritrix3 by internetarchive.

the class CrawlControllerTest method makeTempCrawlController.

// TODO TESTME
public static CrawlController makeTempCrawlController() throws Exception {
    String tmpPath = System.getProperty(TEST_TMP_SYSTEM_PROPERTY_NAME);
    if (tmpPath == null) {
        tmpPath = DEFAULT_TEST_TMP_DIR;
    }
    File tmp = new File(tmpPath);
    FileUtils.ensureWriteableDirectory(tmp);
    FileWriter fileWriter = null;
    try {
        fileWriter = new FileWriter(new File(tmp, "seeds.txt"));
        fileWriter.write("http://www.pandemoniummovie.com");
        fileWriter.close();
    } finally {
        ArchiveUtils.closeQuietly(fileWriter);
    }
    File state = new File(tmp, "state");
    FileUtils.ensureWriteableDirectory(state);
    File checkpoints = new File(tmp, "checkpoints");
    FileUtils.ensureWriteableDirectory(checkpoints);
    BdbModule bdb = new BdbModule();
    bdb.setDir(new ConfigPath("test", state.getAbsolutePath()));
    // def.set(bdb, BdbModule.DIR, state.getAbsolutePath());
    bdb.start();
    CrawlController controller = new CrawlController();
    controller.setServerCache(new BdbServerCache());
    controller.start();
    return controller;
}

Example 2 with BdbModule

use of org.archive.bdb.BdbModule in project heritrix3 by internetarchive.

the class BdbModuleTest method testDoCheckpoint.

@SuppressWarnings("unchecked")
public void testDoCheckpoint() throws Exception {
    ConfigPath basePath = new ConfigPath("testBase", getTmpDir().getAbsolutePath());
    ConfigPath bdbDir = new ConfigPath("bdb", "bdb");
    bdbDir.setBase(basePath);
    FileUtils.deleteDirectory(bdbDir.getFile());
    BdbModule bdb = new BdbModule();
    bdb.setDir(bdbDir);
    bdb.start();
    // avoid data from prior runs being mistaken for current run
    int randomFactor = RandomUtils.nextInt();
    ObjectIdentityBdbManualCache<IdentityCacheableWrapper> testData = bdb.getOIBCCache("testData", false, IdentityCacheableWrapper.class);
    for (int i1 = 0; i1 < 1000; i1++) {
        String key = String.valueOf(i1);
        final String value = String.valueOf(randomFactor * i1);
        String cached = (String) testData.getOrUse(key, new Supplier<IdentityCacheableWrapper>(new IdentityCacheableWrapper(key, value))).get();
        assertSame("unexpected prior entry", value, cached);
    }
    Checkpoint checkpointInProgress = new Checkpoint();
    ConfigPath checkpointsPath = new ConfigPath("checkpoints", "checkpoints");
    checkpointsPath.setBase(basePath);
    checkpointInProgress.generateFrom(checkpointsPath, 998);
    bdb.doCheckpoint(checkpointInProgress);
    String firstCheckpointName = checkpointInProgress.getName();
    for (int i2 = 1000; i2 < 2000; i2++) {
        String key = String.valueOf(i2);
        final String value = String.valueOf(randomFactor * i2);
        String cached = (String) testData.getOrUse(key, new Supplier<IdentityCacheableWrapper>(new IdentityCacheableWrapper(key, value))).get();
        assertSame("unexpected prior entry", value, cached);
    }
    checkpointInProgress = new Checkpoint();
    checkpointInProgress.generateFrom(checkpointsPath, 999);
    bdb.doCheckpoint(checkpointInProgress);
    bdb.stop();
    bdb.destroy();
    BdbModule bdb2 = new BdbModule();
    bdb2.setDir(bdbDir);
    Checkpoint recoveryCheckpoint = new Checkpoint();
    ConfigPath recoverPath = new ConfigPath("recover", firstCheckpointName);
    recoverPath.setBase(basePath);
    recoveryCheckpoint.setCheckpointDir(recoverPath);
    recoveryCheckpoint.afterPropertiesSet();
    bdb2.setRecoveryCheckpoint(recoveryCheckpoint);
    bdb2.start();
    ObjectIdentityBdbManualCache<IdentityCacheableWrapper> restoreData = bdb2.getOIBCCache("testData", true, IdentityCacheableWrapper.class);
    assertEquals("unexpected size", 1000, restoreData.size());
    assertEquals("unexpected value", randomFactor * 999, Integer.parseInt((String) restoreData.get("" + 999).get()));
    bdb2.stop();
    bdb2.destroy();
}

Example 3 with BdbModule

use of org.archive.bdb.BdbModule in project heritrix3 by internetarchive.

the class ContentDigestHistoryTest method bdb.

protected BdbModule bdb() throws IOException {
    if (bdb == null) {
        ConfigPath basePath = new ConfigPath("testBase", getTmpDir().getAbsolutePath());
        ConfigPath bdbDir = new ConfigPath("bdb", "bdb");
        bdbDir.setBase(basePath);
        FileUtils.deleteDirectory(bdbDir.getFile());
        bdb = new BdbModule();
        bdb.setDir(bdbDir);
        bdb.start();
        logger.info("created " + bdb);
    }
    return bdb;
}

Example 4 with BdbModule

use of org.archive.bdb.BdbModule in project heritrix3 by internetarchive.

the class CookieStoreTest method bdb.

protected BdbModule bdb() throws IOException {
    if (bdb == null) {
        ConfigPath basePath = new ConfigPath("testBase", getTmpDir().getAbsolutePath());
        ConfigPath bdbDir = new ConfigPath("bdb", "bdb");
        bdbDir.setBase(basePath);
        FileUtils.deleteDirectory(bdbDir.getFile());
        bdb = new BdbModule();
        bdb.setDir(bdbDir);
        bdb.start();
        logger.info("created " + bdb);
    }
    return bdb;
}