Examples with Environment com.sleepycat.je.Environment used on opensource projects

Example 21 with Environment

use of com.sleepycat.je.Environment in project voldemort by voldemort.

the class BdbStorageConfiguration method removeStorageEngine.

/**
 * Clean up the environment object for the given storage engine
 */
@Override
public void removeStorageEngine(StorageEngine<ByteArray, byte[], byte[]> engine) {
    String storeName = engine.getName();
    BdbStorageEngine bdbEngine = (BdbStorageEngine) engine;
    synchronized (lock) {
        // cleanup a shared 'Environment' object
        if (useOneEnvPerStore) {
            Environment environment = this.environments.get(storeName);
            if (environment == null) {
                // Nothing to clean up.
                return;
            }
            // Remove from the set of unreserved stores if needed.
            if (this.unreservedStores.remove(environment)) {
                logger.info("Removed environment for store name: " + storeName + " from unreserved stores");
            } else {
                logger.info("No environment found in unreserved stores for store name: " + storeName);
            }
            // Try to delete the BDB directory associated
            File bdbDir = environment.getHome();
            if (bdbDir.exists() && bdbDir.isDirectory()) {
                String bdbDirPath = bdbDir.getPath();
                try {
                    FileUtils.deleteDirectory(bdbDir);
                    logger.info("Successfully deleted BDB directory : " + bdbDirPath + " for store name: " + storeName);
                } catch (IOException e) {
                    logger.error("Unable to delete BDB directory: " + bdbDirPath + " for store name: " + storeName);
                }
            }
            // Remove the reference to BdbEnvironmentStats, which holds a
            // reference to the Environment
            BdbEnvironmentStats bdbEnvStats = bdbEngine.getBdbEnvironmentStats();
            this.aggBdbStats.unTrackEnvironment(bdbEnvStats);
            // Unregister the JMX bean for Environment
            if (voldemortConfig.isJmxEnabled()) {
                ObjectName name = JmxUtils.createObjectName(JmxUtils.getPackageName(bdbEnvStats.getClass()), storeName);
                // Un-register the environment stats mbean
                JmxUtils.unregisterMbean(name);
            }
            // Cleanup the environment
            environment.close();
            this.environments.remove(storeName);
            logger.info("Successfully closed the environment for store name : " + storeName);
        }
    }
}

Example 22 with Environment

use of com.sleepycat.je.Environment in project voldemort by voldemort.

the class BdbStorageConfiguration method getEnvironment.

public Environment getEnvironment(StoreDefinition storeDef) throws DatabaseException {
    String storeName = storeDef.getName();
    synchronized (lock) {
        if (useOneEnvPerStore) {
            // reference
            if (environments.containsKey(storeName))
                return environments.get(storeName);
            // otherwise create a new environment
            File bdbDir = new File(bdbMasterDir, storeName);
            createBdbDirIfNecessary(bdbDir);
            // configure the BDB cache
            if (storeDef.hasMemoryFootprint()) {
                // make room for the reservation, by adjusting other stores
                long reservedBytes = storeDef.getMemoryFootprintMB() * ByteUtils.BYTES_PER_MB;
                long newReservedCacheSize = this.reservedCacheSize + reservedBytes;
                // check that we leave a 'minimum' shared cache
                if ((voldemortConfig.getBdbCacheSize() - newReservedCacheSize) < voldemortConfig.getBdbMinimumSharedCache()) {
                    throw new StorageInitializationException("Reservation of " + storeDef.getMemoryFootprintMB() + " MB for store " + storeName + " violates minimum shared cache size of " + voldemortConfig.getBdbMinimumSharedCache());
                }
                this.reservedCacheSize = newReservedCacheSize;
                adjustCacheSizes();
                environmentConfig.setSharedCache(false);
                environmentConfig.setCacheSize(reservedBytes);
            } else {
                environmentConfig.setSharedCache(true);
                environmentConfig.setCacheSize(voldemortConfig.getBdbCacheSize() - this.reservedCacheSize);
            }
            Environment environment = new Environment(bdbDir, environmentConfig);
            logger.info("Creating environment for " + storeName + ": ");
            logEnvironmentConfig(environment.getConfig());
            environments.put(storeName, environment);
            // save this up so we can adjust later if needed
            if (!storeDef.hasMemoryFootprint())
                this.unreservedStores.add(environment);
            return environment;
        } else {
            if (!environments.isEmpty())
                return environments.get(SHARED_ENV_KEY);
            File bdbDir = new File(bdbMasterDir);
            createBdbDirIfNecessary(bdbDir);
            Environment environment = new Environment(bdbDir, environmentConfig);
            logger.info("Creating shared BDB environment: ");
            logEnvironmentConfig(environment.getConfig());
            environments.put(SHARED_ENV_KEY, environment);
            return environment;
        }
    }
}

Example 23 with Environment

use of com.sleepycat.je.Environment in project voldemort by voldemort.

the class BdbStorageConfiguration method getStats.

public String getStats(String storeName, boolean fast) {
    try {
        if (environments.containsKey(storeName)) {
            StatsConfig config = new StatsConfig();
            config.setFast(fast);
            Environment env = environments.get(storeName);
            return env.getStats(config).toString();
        } else {
            // return empty string if environment not created yet
            return "";
        }
    } catch (DatabaseException e) {
        throw new VoldemortException(e);
    }
}

Example 24 with Environment

use of com.sleepycat.je.Environment in project voldemort by voldemort.

the class BdbGrowth method main.

public static void main(String[] args) throws Exception {
    if (args.length != 5) {
        System.err.println("USAGE: java BdbGrowth directory cache_size total_size increment threads");
        System.exit(1);
    }
    final String dir = args[0];
    final long cacheSize = Long.parseLong(args[1]);
    final int totalSize = Integer.parseInt(args[2]);
    final int increment = Integer.parseInt(args[3]);
    final int threads = Integer.parseInt(args[4]);
    Environment environment;
    EnvironmentConfig environmentConfig;
    DatabaseConfig databaseConfig;
    environmentConfig = new EnvironmentConfig();
    environmentConfig.setCacheSize(cacheSize);
    environmentConfig.setDurability(Durability.COMMIT_NO_SYNC);
    environmentConfig.setConfigParam(EnvironmentConfig.LOG_FILE_MAX, "1000000000");
    environmentConfig.setConfigParam(EnvironmentConfig.CLEANER_MAX_BATCH_FILES, "100");
    environmentConfig.setConfigParam(EnvironmentConfig.CLEANER_READ_SIZE, "52428800");
    environmentConfig.setAllowCreate(true);
    environmentConfig.setTransactional(true);
    databaseConfig = new DatabaseConfig();
    databaseConfig.setAllowCreate(true);
    // databaseConfig.setDeferredWrite(true);
    databaseConfig.setTransactional(true);
    databaseConfig.setNodeMaxEntries(1024);
    File bdbDir = new File(dir);
    if (!bdbDir.exists()) {
        bdbDir.mkdir();
    } else {
        for (File f : bdbDir.listFiles()) f.delete();
    }
    environment = new Environment(bdbDir, environmentConfig);
    final Database db = environment.openDatabase(null, "test", databaseConfig);
    final Random rand = new Random();
    int iterations = totalSize / increment;
    long[] readTimes = new long[iterations];
    long[] writeTimes = new long[iterations];
    ExecutorService service = Executors.newFixedThreadPool(threads);
    for (int i = 0; i < iterations; i++) {
        System.out.println("Starting iteration " + i);
        List<Future<Object>> results = new ArrayList<Future<Object>>(increment);
        long startTime = System.currentTimeMillis();
        final int fi = i;
        for (int j = 0; j < increment; j++) {
            final int fj = j;
            results.add(service.submit(new Callable<Object>() {

                public Object call() throws Exception {
                    db.put(null, new DatabaseEntry(Integer.toString(fi * increment + fj).getBytes()), new DatabaseEntry(Integer.toString(fi * increment + fj).getBytes()));
                    return null;
                }
            }));
        }
        for (int j = 0; j < increment; j++) results.get(j).get();
        writeTimes[i] = System.currentTimeMillis() - startTime;
        System.out.println("write: " + (writeTimes[i] / (double) increment));
        results.clear();
        startTime = System.currentTimeMillis();
        for (int j = 0; j < increment; j++) {
            results.add(service.submit(new Callable<Object>() {

                public Object call() throws Exception {
                    int value = rand.nextInt((fi + 1) * increment);
                    return db.get(null, new DatabaseEntry(Integer.toString(value).getBytes()), new DatabaseEntry(Integer.toString(value).getBytes()), null);
                }
            }));
        }
        for (int j = 0; j < increment; j++) results.get(j).get();
        readTimes[i] = (System.currentTimeMillis() - startTime);
        System.out.println("read: " + (readTimes[i] / (double) increment));
        int cleaned = 0;
        do {
            cleaned += environment.cleanLog();
        } while (cleaned > 0);
        if (cleaned > 0)
            System.out.println("Cleaned " + cleaned + " files.");
        CheckpointConfig cp = new CheckpointConfig();
        cp.setForce(true);
        environment.checkpoint(null);
        environment.compress();
        environment.sync();
        System.out.println("Cleaning, Checkpointing and compression completed.");
    }
    System.out.println();
    System.out.println("iteration read write:");
    for (int i = 0; i < iterations; i++) {
        System.out.print(i);
        System.out.print(" " + readTimes[i] / (double) increment);
        System.out.println(" " + writeTimes[i] / (double) increment);
    }
    System.out.println(environment.getStats(null));
    System.exit(0);
}

Example 25 with Environment

use of com.sleepycat.je.Environment in project voldemort by voldemort.

the class BdbStorageEngineTest method testPersistence.

@Test
public void testPersistence() throws Exception {
    this.store.put(new ByteArray("abc".getBytes()), new Versioned<byte[]>("cdef".getBytes()), null);
    this.store.close();
    this.environment.close();
    this.environment = new Environment(this.tempDir, envConfig);
    this.database = environment.openDatabase(null, "test", databaseConfig);
    this.store = makeBdbStorageEngine("test", this.environment, this.database, runtimeConfig, this.prefixPartitionId);
    List<Versioned<byte[]>> vals = store.get(new ByteArray("abc".getBytes()), null);
    assertEquals(1, vals.size());
    TestUtils.bytesEqual("cdef".getBytes(), vals.get(0).getValue());
}