Examples with StorageInitializationException voldemort.store.StorageInitializationException used on opensource projects

Example 1 with StorageInitializationException

use of voldemort.store.StorageInitializationException in project voldemort by voldemort.

the class RocksDbStorageConfiguration method getStore.

@Override
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String storeName = storeDef.getName();
    if (!stores.containsKey(storeName)) {
        String dataDir = this.voldemortconfig.getRdbDataDirectory() + "/" + storeName;
        new File(dataDir).mkdirs();
        Properties dbProperties = parseProperties(VoldemortConfig.ROCKSDB_DB_OPTIONS);
        DBOptions dbOptions = (dbProperties.size() > 0) ? DBOptions.getDBOptionsFromProps(dbProperties) : new DBOptions();
        if (dbOptions == null) {
            throw new StorageInitializationException("Unable to parse Data Base Options.");
        }
        dbOptions.setCreateIfMissing(true);
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.createStatistics();
        Properties cfProperties = parseProperties(VoldemortConfig.ROCKSDB_CF_OPTIONS);
        if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
            cfProperties.setProperty("prefix_extractor", "fixed:" + StoreBinaryFormat.PARTITIONID_PREFIX_SIZE);
        }
        ColumnFamilyOptions cfOptions = (cfProperties.size() > 0) ? ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfProperties) : new ColumnFamilyOptions();
        if (cfOptions == null) {
            throw new StorageInitializationException("Unable to parse Column Family Options.");
        }
        // Create a non default Column Family tp hold the store data.
        List<ColumnFamilyDescriptor> descriptors = new ArrayList<ColumnFamilyDescriptor>();
        descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
        descriptors.add(new ColumnFamilyDescriptor(storeName.getBytes(), cfOptions));
        List<ColumnFamilyHandle> handles = new ArrayList<ColumnFamilyHandle>();
        try {
            RocksDB rdbStore = RocksDB.open(dbOptions, dataDir, descriptors, handles);
            // Dispose of the default Column Family immediately.  We don't use it and if it has not been disposed
            // by the time the DB is closed then the RocksDB code can terminate abnormally (if the RocksDB code is
            // built with assertions enabled). The handle will go out of scope on its own and the Java finalizer
            // will (eventually) do this for us, but, that is not fast enough for the unit tests.
            handles.get(0).dispose();
            ColumnFamilyHandle storeHandle = handles.get(1);
            RocksDbStorageEngine rdbStorageEngine;
            if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
                rdbStorageEngine = new PartitionPrefixedRocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, strategy, voldemortconfig.isRocksdbEnableReadLocks());
            } else {
                rdbStorageEngine = new RocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, voldemortconfig.isRocksdbEnableReadLocks());
            }
            stores.put(storeName, rdbStorageEngine);
        } catch (Exception e) {
            throw new StorageInitializationException(e);
        }
    }
    return stores.get(storeName);
}

Example 2 with StorageInitializationException

use of voldemort.store.StorageInitializationException in project voldemort by voldemort.

the class BdbStorageConfiguration method update.

/**
     * Detect what has changed in the store definition and rewire BDB
     * environments accordingly.
     * 
     * @param storeDef updated store definition
     */
public void update(StoreDefinition storeDef) {
    if (!useOneEnvPerStore)
        throw new VoldemortException("Memory foot print can be set only when using different environments per store");
    String storeName = storeDef.getName();
    Environment environment = environments.get(storeName);
    // change reservation amount of reserved store
    if (!unreservedStores.contains(environment) && storeDef.hasMemoryFootprint()) {
        EnvironmentMutableConfig mConfig = environment.getMutableConfig();
        long currentCacheSize = mConfig.getCacheSize();
        long newCacheSize = storeDef.getMemoryFootprintMB() * ByteUtils.BYTES_PER_MB;
        if (currentCacheSize != newCacheSize) {
            long newReservedCacheSize = this.reservedCacheSize - currentCacheSize + newCacheSize;
            // check that we leave a 'minimum' shared cache
            if ((voldemortConfig.getBdbCacheSize() - newReservedCacheSize) < voldemortConfig.getBdbMinimumSharedCache()) {
                throw new StorageInitializationException("Reservation of " + storeDef.getMemoryFootprintMB() + " MB for store " + storeName + " violates minimum shared cache size of " + voldemortConfig.getBdbMinimumSharedCache());
            }
            this.reservedCacheSize = newReservedCacheSize;
            adjustCacheSizes();
            mConfig.setCacheSize(newCacheSize);
            environment.setMutableConfig(mConfig);
            logger.info("Setting private cache for store " + storeDef.getName() + " to " + newCacheSize);
        }
    } else {
        // versa since the sharedCache param is not mutable
        throw new VoldemortException("Cannot switch between shared and private cache dynamically");
    }
}

Example 3 with StorageInitializationException

use of voldemort.store.StorageInitializationException in project voldemort by voldemort.

the class BdbStorageConfiguration method getStore.

public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    synchronized (lock) {
        try {
            String storeName = storeDef.getName();
            Environment environment = getEnvironment(storeDef);
            Database db = environment.openDatabase(null, storeName, databaseConfig);
            BdbRuntimeConfig runtimeConfig = new BdbRuntimeConfig(voldemortConfig);
            BdbStorageEngine engine = null;
            if (voldemortConfig.getBdbPrefixKeysWithPartitionId()) {
                engine = new PartitionPrefixedBdbStorageEngine(storeName, environment, db, runtimeConfig, strategy);
            } else {
                engine = new BdbStorageEngine(storeName, environment, db, runtimeConfig);
            }
            if (voldemortConfig.isJmxEnabled()) {
                // register the environment stats mbean
                JmxUtils.registerMbean(storeName, engine.getBdbEnvironmentStats());
                // aggregated stats
                if (useOneEnvPerStore) {
                    aggBdbStats.trackEnvironment(engine.getBdbEnvironmentStats());
                }
            }
            return engine;
        } catch (DatabaseException d) {
            throw new StorageInitializationException(d);
        }
    }
}

Example 4 with StorageInitializationException

use of voldemort.store.StorageInitializationException in project voldemort by voldemort.

the class BdbStorageConfiguration method getEnvironment.

public Environment getEnvironment(StoreDefinition storeDef) throws DatabaseException {
    String storeName = storeDef.getName();
    synchronized (lock) {
        if (useOneEnvPerStore) {
            // reference
            if (environments.containsKey(storeName))
                return environments.get(storeName);
            // otherwise create a new environment
            File bdbDir = new File(bdbMasterDir, storeName);
            createBdbDirIfNecessary(bdbDir);
            // configure the BDB cache
            if (storeDef.hasMemoryFootprint()) {
                // make room for the reservation, by adjusting other stores
                long reservedBytes = storeDef.getMemoryFootprintMB() * ByteUtils.BYTES_PER_MB;
                long newReservedCacheSize = this.reservedCacheSize + reservedBytes;
                // check that we leave a 'minimum' shared cache
                if ((voldemortConfig.getBdbCacheSize() - newReservedCacheSize) < voldemortConfig.getBdbMinimumSharedCache()) {
                    throw new StorageInitializationException("Reservation of " + storeDef.getMemoryFootprintMB() + " MB for store " + storeName + " violates minimum shared cache size of " + voldemortConfig.getBdbMinimumSharedCache());
                }
                this.reservedCacheSize = newReservedCacheSize;
                adjustCacheSizes();
                environmentConfig.setSharedCache(false);
                environmentConfig.setCacheSize(reservedBytes);
            } else {
                environmentConfig.setSharedCache(true);
                environmentConfig.setCacheSize(voldemortConfig.getBdbCacheSize() - this.reservedCacheSize);
            }
            Environment environment = new Environment(bdbDir, environmentConfig);
            logger.info("Creating environment for " + storeName + ": ");
            logEnvironmentConfig(environment.getConfig());
            environments.put(storeName, environment);
            // save this up so we can adjust later if needed
            if (!storeDef.hasMemoryFootprint())
                this.unreservedStores.add(environment);
            return environment;
        } else {
            if (!environments.isEmpty())
                return environments.get(SHARED_ENV_KEY);
            File bdbDir = new File(bdbMasterDir);
            createBdbDirIfNecessary(bdbDir);
            Environment environment = new Environment(bdbDir, environmentConfig);
            logger.info("Creating shared BDB environment: ");
            logEnvironmentConfig(environment.getConfig());
            environments.put(SHARED_ENV_KEY, environment);
            return environment;
        }
    }
}

Example 5 with StorageInitializationException

use of voldemort.store.StorageInitializationException in project voldemort by voldemort.

the class BdbCachePartitioningTest method testMinimumSharedCache.

/**
     * Tests that any reservation that will not violate minimum shared cache
     * will fail, during server startup and dynamic updation
     */
@Test
public void testMinimumSharedCache() {
    // total cache size
    int totalCache = 20 * ByteUtils.BYTES_PER_MB;
    // A reserves 10MB
    int shareA = 10 * ByteUtils.BYTES_PER_MB;
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(totalCache);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
    voldemortConfig.setBdbMinimumSharedCache(15 * ByteUtils.BYTES_PER_MB);
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    BdbStorageEngine storeA = null;
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    assertEquals("Reserved cache size not zero", 0, bdbStorage.getReservedCacheSize());
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
        storeA = (BdbStorageEngine) bdbStorage.getStore(defA, TestUtils.makeSingleNodeRoutingStrategy());
        fail("Should have thrown exception since minSharedCache will be violated");
    } catch (StorageInitializationException sie) {
    // should come here.
    }
    // failing operations should not alter reserved cache size
    assertEquals("failure somehow altered the reservedCacheSize", 0, bdbStorage.getReservedCacheSize());
    voldemortConfig.setBdbMinimumSharedCache(10 * ByteUtils.BYTES_PER_MB);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
        storeA = (BdbStorageEngine) bdbStorage.getStore(defA, TestUtils.makeSingleNodeRoutingStrategy());
    } catch (StorageInitializationException sie) {
        // should not come here.
        fail("minSharedCache should n't have been violated");
    }
    assertEquals("store A's share does not match up with reserved cache size", shareA, bdbStorage.getReservedCacheSize());
    long reserveCacheSize = bdbStorage.getReservedCacheSize();
    // now, try increasing the reservation dynamically and it should fail
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", 15);
        bdbStorage.update(defA);
        fail("Should have thrown exception since minSharedCache will be violated");
    } catch (StorageInitializationException sie) {
    // should come here.
    }
    assertEquals("failure somehow altered the reservedCacheSize", reserveCacheSize, bdbStorage.getReservedCacheSize());
    if (storeA != null)
        storeA.close();
    bdbStorage.close();
}