Examples with StorageConfiguration voldemort.store.StorageConfiguration used on opensource projects

Example 1 with StorageConfiguration

use of voldemort.store.StorageConfiguration in project voldemort by voldemort.

the class StorageEnginePerformanceTest method main.

public static void main(String[] args) throws Exception {
    try {
        OptionParser parser = new OptionParser();
        parser.accepts("help", "print usage information");
        parser.accepts("requests", "[REQUIRED] number of requests to execute").withRequiredArg().ofType(Integer.class);
        parser.accepts("num-values", "[REQUIRED] number of values in the store").withRequiredArg().ofType(Integer.class);
        parser.accepts("data-dir", "Data directory for storage data").withRequiredArg().describedAs("directory");
        parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
        parser.accepts("storage-configuration-class", "[REQUIRED] class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class_name");
        parser.accepts("props", "Properties file with configuration for the engine").withRequiredArg().describedAs("config.properties");
        parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
        parser.accepts("cache-width", "Percentage of requests to save as possible re-requests").withRequiredArg().describedAs("width").ofType(Integer.class);
        parser.accepts("cache-hit-ratio", "Percentage of requests coming from the last cache-width requests").withRequiredArg().describedAs("ratio").ofType(Double.class);
        parser.accepts("clean-up", "Delete data directory when done.");
        OptionSet options = parser.parse(args);
        if (options.has("help")) {
            parser.printHelpOn(System.out);
            System.exit(0);
        }
        CmdUtils.croakIfMissing(parser, options, "requests");
        final int numThreads = CmdUtils.valueOf(options, "threads", 10);
        final int numRequests = (Integer) options.valueOf("requests");
        final int numValues = (Integer) options.valueOf("num-values");
        final int valueSize = CmdUtils.valueOf(options, "value-size", 1024);
        final int cacheWidth = CmdUtils.valueOf(options, "cache-width", 100000);
        final double cacheHitRatio = CmdUtils.valueOf(options, "cache-hit-ratio", 0.5);
        final String propsFile = (String) options.valueOf("props");
        final boolean cleanUp = options.has("clean-up");
        final String storageEngineClass = CmdUtils.valueOf(options, "storage-configuration-class", BdbStorageConfiguration.class.getName()).trim();
        File dataDir = null;
        if (options.has("data-dir"))
            dataDir = new File((String) options.valueOf("data-dir"));
        else
            dataDir = TestUtils.createTempDir();
        System.out.println("Data dir: " + dataDir);
        // create the storage engine
        Props props = new Props();
        if (propsFile != null)
            props = new Props(new File(propsFile));
        props.put("node.id", 0);
        props.put("data.directory", dataDir.getAbsolutePath());
        props.put("voldemort.home", System.getProperty("user.dir"));
        VoldemortConfig config = new VoldemortConfig(props);
        StorageConfiguration storageConfig = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { config });
        StorageEngine<ByteArray, byte[], byte[]> engine = storageConfig.getStore(TestUtils.makeStoreDefinition("test"), TestUtils.makeSingleNodeRoutingStrategy());
        @SuppressWarnings("unchecked") final Store<String, byte[], byte[]> store = new SerializingStore(engine, new StringSerializer(), new IdentitySerializer(), null);
        final byte[] value = new byte[valueSize];
        new Random().nextBytes(value);
        // initialize test data
        for (int i = 0; i < numValues; i++) store.put(Integer.toString(i), Versioned.value(value), null);
        // initialize cache lookback data
        int[] recents = new int[cacheWidth];
        System.out.println("Write test:");
        CachedPerformanceTest writeTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                try {
                    String key = Integer.toString(index);
                    List<Versioned<byte[]>> vs = store.get(key, null);
                    VectorClock version;
                    if (vs.size() == 0)
                        version = new VectorClock();
                    else
                        version = (VectorClock) vs.get(0).getVersion();
                    version.incrementVersion(0, 847584375);
                    store.put(key, Versioned.value(value, version), null);
                } catch (ObsoleteVersionException e) {
                // do nothing
                } catch (RuntimeException e) {
                    e.printStackTrace();
                    throw e;
                }
            }
        }, recents, numValues, cacheHitRatio);
        writeTest.run(numRequests, numThreads);
        writeTest.printStats();
        System.out.println();
        System.out.println("Read test:");
        CachedPerformanceTest readTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                store.get(Integer.toString(index), null);
            }
        }, recents, numValues, cacheHitRatio);
        readTest.run(numRequests, numThreads);
        readTest.printStats();
        if (cleanUp)
            Utils.rm(dataDir);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }
}

Example 2 with StorageConfiguration

use of voldemort.store.StorageConfiguration in project voldemort by voldemort.

the class Benchmark method initializeStore.

@SuppressWarnings("unchecked")
public void initializeStore(Props benchmarkProps) throws Exception {
    this.numThreads = benchmarkProps.getInt(THREADS, MAX_WORKERS);
    this.numConnectionsPerNode = benchmarkProps.getInt(NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE);
    this.numIterations = benchmarkProps.getInt(ITERATIONS, 1);
    this.statusIntervalSec = benchmarkProps.getInt(INTERVAL, 0);
    this.verbose = benchmarkProps.getBoolean(VERBOSE, false);
    this.verifyRead = benchmarkProps.getBoolean(VERIFY, false);
    this.ignoreNulls = benchmarkProps.getBoolean(IGNORE_NULLS, false);
    int clientZoneId = benchmarkProps.getInt(CLIENT_ZONE_ID, -1);
    if (benchmarkProps.containsKey(URL)) {
        // Remote benchmark
        if (!benchmarkProps.containsKey(STORE_NAME)) {
            throw new VoldemortException("Missing storename");
        }
        String socketUrl = benchmarkProps.getString(URL);
        String storeName = benchmarkProps.getString(STORE_NAME);
        ClientConfig clientConfig = new ClientConfig().setMaxThreads(numThreads).setMaxTotalConnections(numThreads).setMaxConnectionsPerNode(numConnectionsPerNode).setRoutingTimeout(1500, TimeUnit.MILLISECONDS).setSocketTimeout(1500, TimeUnit.MILLISECONDS).setConnectionTimeout(500, TimeUnit.MILLISECONDS).setRequestFormatType(RequestFormatType.VOLDEMORT_V3).setBootstrapUrls(socketUrl);
        if (clientZoneId >= 0) {
            clientConfig.setClientZoneId(clientZoneId);
        }
        SocketStoreClientFactory socketFactory = new SocketStoreClientFactory(clientConfig);
        this.storeClient = socketFactory.getStoreClient(storeName);
        StoreDefinition storeDef = getStoreDefinition(socketFactory, storeName);
        this.keyType = findKeyType(storeDef);
        benchmarkProps.put(Benchmark.KEY_TYPE, this.keyType);
        this.factory = socketFactory;
    } else {
        // Local benchmark
        localMode = true;
        String storageEngineClass = benchmarkProps.getString(STORAGE_CONFIGURATION_CLASS);
        this.keyType = benchmarkProps.getString(KEY_TYPE, STRING_KEY_TYPE);
        Serializer serializer = findKeyType(this.keyType);
        Store<Object, Object, Object> store = null;
        VoldemortConfig voldemortConfig;
        if (benchmarkProps.containsKey(LOCAL_SERVER_PROPERTIES)) {
            File homeDir = TestUtils.createTempDir();
            File configDir = new File(homeDir, "config");
            configDir.mkdir();
            FileUtils.copyFile(new File(benchmarkProps.get(LOCAL_SERVER_PROPERTIES)), new File(configDir, "server.properties"));
            voldemortConfig = VoldemortConfig.loadFromVoldemortHome(homeDir.getAbsolutePath());
        } else {
            voldemortConfig = ServerTestUtils.getVoldemortConfig();
        }
        StorageConfiguration conf = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { voldemortConfig });
        StorageEngine<ByteArray, byte[], byte[]> engine = conf.getStore(TestUtils.makeStoreDefinition(DUMMY_DB), TestUtils.makeSingleNodeRoutingStrategy());
        if (conf.getType().compareTo(ViewStorageConfiguration.TYPE_NAME) == 0) {
            engine = new ViewStorageEngine(STORE_NAME, engine, new StringSerializer(), new StringSerializer(), serializer, new StringSerializer(), null, BenchmarkViews.loadTransformation(benchmarkProps.getString(VIEW_CLASS).trim()));
        }
        store = SerializingStore.wrap(engine, serializer, new StringSerializer(), new IdentitySerializer());
        this.factory = new StaticStoreClientFactory(store);
        this.storeClient = factory.getStoreClient(store.getName());
    }
    this.storeInitialized = true;
}

Example 3 with StorageConfiguration

use of voldemort.store.StorageConfiguration in project voldemort by voldemort.

the class StorageService method removeEngine.

/**
     * Unregister and remove the engine from the storage repository. This is
     * called during deletion of stores and if there are exceptions
     * adding/opening stores
     * 
     * @param engine The actual engine to remove
     * @param isReadOnly Is this read-only?
     * @param storeType The storage type of the store
     * @param truncate Should the store be truncated?
     */
public void removeEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, boolean truncate) {
    String storeName = engine.getName();
    Store<ByteArray, byte[], byte[]> store = storeRepository.removeLocalStore(storeName);
    boolean isSlop = storeType.compareTo("slop") == 0;
    boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
    boolean isMetadata = storeName.compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
    if (store != null) {
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            if (!isSlop && voldemortConfig.isEnableRebalanceService() && !isReadOnly && !isMetadata && !isView) {
                ObjectName name = null;
                if (this.voldemortConfig.isEnableJmxClusterName())
                    name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                else
                    name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                synchronized (mbeanServer) {
                    if (mbeanServer.isRegistered(name))
                        JmxUtils.unregisterMbean(mbeanServer, name);
                }
            }
            if (voldemortConfig.isStatTrackingEnabled()) {
                ObjectName name = null;
                if (this.voldemortConfig.isEnableJmxClusterName())
                    name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
                else
                    name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
                synchronized (mbeanServer) {
                    if (mbeanServer.isRegistered(name))
                        JmxUtils.unregisterMbean(mbeanServer, name);
                }
            }
        }
        if (voldemortConfig.isServerRoutingEnabled() && !isSlop) {
            this.storeRepository.removeRoutedStore(storeName);
            for (Node node : metadata.getCluster().getNodes()) this.storeRepository.removeNodeStore(storeName, node.getId());
        }
    }
    storeRepository.removeStorageEngine(storeName);
    // Then truncate (if needed) and close
    if (truncate) {
        engine.truncate();
    }
    engine.close();
    // Also remove any state in the StorageConfiguration (if required)
    StorageConfiguration config = storageConfigs.get(storeType);
    if (config == null) {
        throw new ConfigurationException("Attempt to close storage engine " + engine.getName() + " but " + storeType + " storage engine has not been enabled.");
    }
    config.removeStorageEngine(engine);
}

Example 4 with StorageConfiguration

use of voldemort.store.StorageConfiguration in project voldemort by voldemort.

the class StorageService method stopInner.

@Override
protected void stopInner() {
    /*
         * We may end up closing a given store more than once, but that is cool
         * because close() is idempotent
         */
    Exception lastException = null;
    logger.info("Closing all stores.");
    /* This will also close the node stores including local stores */
    for (Store<ByteArray, byte[], byte[]> store : this.storeRepository.getAllRoutedStores()) {
        logger.info("Closing routed store for " + store.getName());
        try {
            store.close();
        } catch (Exception e) {
            logger.error(e);
            lastException = e;
        }
    }
    /* This will also close the storage engines */
    for (Store<ByteArray, byte[], byte[]> store : this.storeRepository.getAllStorageEngines()) {
        logger.info("Closing storage engine for " + store.getName());
        try {
            store.close();
        } catch (Exception e) {
            logger.error(e);
            lastException = e;
        }
    }
    logger.info("All stores closed.");
    /* Close slop store if necessary */
    if (this.storeRepository.hasSlopStore()) {
        try {
            this.storeRepository.getSlopStore().close();
        } catch (Exception e) {
            logger.error(e);
            lastException = e;
        }
    }
    /* Close all storage configs */
    logger.info("Closing storage configurations.");
    for (StorageConfiguration config : storageConfigs.values()) {
        logger.info("Closing " + config.getType() + " storage config.");
        try {
            config.close();
        } catch (Exception e) {
            logger.error(e);
            lastException = e;
        }
    }
    this.clientThreadPool.shutdown();
    try {
        if (!this.clientThreadPool.awaitTermination(10, TimeUnit.SECONDS))
            this.clientThreadPool.shutdownNow();
    } catch (InterruptedException e) {
        // okay, fine, playing nice didn't work
        this.clientThreadPool.shutdownNow();
    }
    logger.info("Closed client threadpool.");
    storeFactory.close();
    if (this.failureDetector != null) {
        try {
            this.failureDetector.destroy();
        } catch (Exception e) {
            lastException = e;
        }
    }
    logger.info("Closed failure detector.");
    // shut down the proxy put thread pool
    this.proxyPutWorkerPool.shutdown();
    try {
        if (!this.proxyPutWorkerPool.awaitTermination(10, TimeUnit.SECONDS))
            this.proxyPutWorkerPool.shutdownNow();
    } catch (InterruptedException e) {
        this.proxyPutWorkerPool.shutdownNow();
    }
    logger.info("Closed proxy put thread pool.");
    /* If there is an exception, throw it */
    if (lastException instanceof VoldemortException)
        throw (VoldemortException) lastException;
    else if (lastException != null)
        throw new VoldemortException(lastException);
}

Example 5 with StorageConfiguration

use of voldemort.store.StorageConfiguration in project voldemort by voldemort.

the class StorageService method startInner.

@Override
protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");
    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations()) initStorageConfig(configClassName);
    /* Initialize view storage configuration */
    storageConfigs.put(ViewStorageConfiguration.TYPE_NAME, new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));
    /* Initialize system stores */
    initSystemStores();
    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {
        logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
        StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
        if (config == null)
            throw new ConfigurationException("Attempt to open store " + SlopStorageEngine.SLOP_STORE_NAME + " but " + voldemortConfig.getSlopStoreType() + " storage engine has not been enabled.");
        // make a dummy store definition object
        StoreDefinition slopStoreDefinition = new StoreDefinition(SlopStorageEngine.SLOP_STORE_NAME, null, null, null, null, null, null, RoutingStrategyType.CONSISTENT_STRATEGY, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        SlopStorageEngine slopEngine = new SlopStorageEngine(config.getStore(slopStoreDefinition, new RoutingStrategyFactory().updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())), metadata.getCluster());
        registerInternalEngine(slopEngine, false, "slop");
        storeRepository.setSlopStore(slopEngine);
        if (voldemortConfig.isSlopPusherJobEnabled()) {
            // Now initialize the pusher job after some time
            GregorianCalendar cal = new GregorianCalendar();
            cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
            Date nextRun = cal.getTime();
            logger.info("Initializing slop pusher job type " + voldemortConfig.getPusherType() + " at " + nextRun);
            scheduler.schedule("slop", (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0) ? new BlockingSlopPusherJob(storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper) : new StreamingSlopPusherJob(storeRepository, metadata, slopStreamingFailureDetector, voldemortConfig, scanPermitWrapper), nextRun, voldemortConfig.getSlopFrequencyMs());
        }
        // Create a SlopPurgeJob object and register it
        if (voldemortConfig.isSlopPurgeJobEnabled()) {
            logger.info("Initializing Slop Purge job");
            SlopPurgeJob job = new SlopPurgeJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
            JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
            storeRepository.registerSlopPurgeJob(job);
        }
    }
    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
        logger.info("Initializing repair job.");
        RepairJob job = new RepairJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getRepairJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerRepairJob(job);
    }
    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
        logger.info("Intializing prune job");
        VersionedPutPruneJob job = new VersionedPutPruneJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getPruneJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerPruneJob(job);
    }
    List<StoreDefinition> storeDefs = new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");
    logger.info("Validating schemas:");
    StoreDefinitionUtils.validateSchemasAsNeeded(storeDefs);
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView())
        openStore(def);
    // those stores
    for (StoreDefinition def : storeDefs) {
        if (def.isView())
            openStore(def);
    }
    initializeMetadataVersions(storeDefs);
    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
        if (this.voldemortConfig.isEnableJmxClusterName())
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName(metadata.getCluster().getName() + ".voldemort.store.stats.aggregate", "aggregate-perf"));
        else
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));
    List<StorageEngine> listOfDisabledStores = Lists.newArrayList();
    for (StorageEngine storageEngine : storeRepository.getAllStorageEngines()) {
        try {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            if (storeVersionManager.hasAnyDisabledVersion()) {
                listOfDisabledStores.add(storageEngine);
                logger.warn("The following store is marked as disabled: " + storageEngine.getName());
            // Must put server in offline mode.
            }
        } catch (NoSuchCapabilityException e) {
        // Not a read-only store: no-op
        }
    }
    if (listOfDisabledStores.isEmpty()) {
        logger.info("All stores initialized.");
    } else {
        throw new DisabledStoreException("All stores initialized, but the server needs to go " + "in offline mode because some store(s) are disabled.");
    }
}