Search in sources :

Example 1 with QuotaLimitingStore

use of voldemort.store.quota.QuotaLimitingStore in project voldemort by voldemort.

the class StorageService method registerEngine.

/**
     * Register the given engine with the storage repository
     * 
     * @param engine Register the storage engine
     * @param isReadOnly Boolean indicating if this store is read-only
     * @param storeType The type of the store
     * @param storeDef store definition for the store to be registered
     */
public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, StoreDefinition storeDef) {
    Cluster cluster = this.metadata.getCluster();
    storeRepository.addStorageEngine(engine);
    /* Now add any store wrappers that are enabled */
    Store<ByteArray, byte[], byte[]> store = engine;
    boolean isMetadata = store.getName().compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
    boolean isSlop = storeType.compareTo("slop") == 0;
    boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
    if (voldemortConfig.isVerboseLoggingEnabled())
        store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
    if (!isSlop) {
        if (!isReadOnly && !isMetadata && !isView) {
            // wrap store to enforce retention policy
            if (voldemortConfig.isEnforceRetentionPolicyOnRead() && storeDef != null) {
                RetentionEnforcingStore retentionEnforcingStore = new RetentionEnforcingStore(store, storeDef, voldemortConfig.isDeleteExpiredValuesOnRead(), SystemTime.INSTANCE);
                metadata.addMetadataStoreListener(store.getName(), retentionEnforcingStore);
                store = retentionEnforcingStore;
            }
            if (voldemortConfig.isEnableRebalanceService()) {
                ProxyPutStats proxyPutStats = new ProxyPutStats(aggregatedProxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    JmxUtils.registerMbean(proxyPutStats, JmxUtils.createObjectName("voldemort.store.rebalancing", engine.getName() + "-proxy-puts"));
                }
                store = new RedirectingStore(store, metadata, storeRepository, failureDetector, storeFactory, proxyPutWorkerPool, proxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
                    ObjectName name = null;
                    if (this.voldemortConfig.isEnableJmxClusterName())
                        name = JmxUtils.createObjectName(cluster.getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    else
                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    synchronized (mbeanServer) {
                        if (mbeanServer.isRegistered(name))
                            JmxUtils.unregisterMbean(mbeanServer, name);
                        JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(store), name);
                    }
                }
            }
        }
        if (voldemortConfig.isMetadataCheckingEnabled() && !isMetadata) {
            store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
        }
    }
    if (voldemortConfig.isStatTrackingEnabled()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
        store = statStore;
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName name = null;
            if (this.voldemortConfig.isEnableJmxClusterName())
                name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
            else
                name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
            synchronized (mbeanServer) {
                if (mbeanServer.isRegistered(name))
                    JmxUtils.unregisterMbean(mbeanServer, name);
                JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
            }
        }
        // metadata store)
        if (voldemortConfig.isEnableQuotaLimiting() && !isMetadata) {
            StoreStats currentStoreStats = statStore.getStats();
            FileBackedCachingStorageEngine quotaStore = (FileBackedCachingStorageEngine) storeRepository.getStorageEngine(SystemStoreConstants.SystemStoreName.voldsys$_store_quotas.toString());
            QuotaLimitStats quotaStats = new QuotaLimitStats(this.aggregatedQuotaStats);
            QuotaLimitingStore rateLimitingStore = new QuotaLimitingStore(store, currentStoreStats, quotaStats, quotaStore, metadata);
            if (voldemortConfig.isJmxEnabled()) {
                JmxUtils.registerMbean(quotaStats, JmxUtils.createObjectName("voldemort.store.quota", store.getName() + "-quota-limit-stats"));
            }
            store = rateLimitingStore;
        }
    }
    storeRepository.addLocalStore(store);
}
Also used : StoreStats(voldemort.store.stats.StoreStats) RedirectingStore(voldemort.store.rebalancing.RedirectingStore) Cluster(voldemort.cluster.Cluster) LoggingStore(voldemort.store.logging.LoggingStore) ObjectName(javax.management.ObjectName) StatTrackingStore(voldemort.store.stats.StatTrackingStore) FileBackedCachingStorageEngine(voldemort.store.configuration.FileBackedCachingStorageEngine) ProxyPutStats(voldemort.store.rebalancing.ProxyPutStats) RetentionEnforcingStore(voldemort.store.retention.RetentionEnforcingStore) ByteArray(voldemort.utils.ByteArray) InvalidMetadataCheckingStore(voldemort.store.invalidmetadata.InvalidMetadataCheckingStore) QuotaLimitingStore(voldemort.store.quota.QuotaLimitingStore) QuotaLimitStats(voldemort.store.quota.QuotaLimitStats) StoreStatsJmx(voldemort.store.stats.StoreStatsJmx) MBeanServer(javax.management.MBeanServer)

Example 2 with QuotaLimitingStore

use of voldemort.store.quota.QuotaLimitingStore in project voldemort by voldemort.

the class QuotaLimitingStoreTest method testQuotaPctUsageCalculation.

@Test
public /**
     *  PS: Test will fail if for some reason we cannot do 50 ops/sec against a hash map. So yeah, pretty unlikely.
     */
void testQuotaPctUsageCalculation() throws Exception {
    File tempDir = TestUtils.createTempDir();
    FileBackedCachingStorageEngine quotaStore = new FileBackedCachingStorageEngine("quota-usage-test-store", tempDir.getAbsolutePath());
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("inMemoryBackingStore");
    QuotaLimitStats quotaStats = new QuotaLimitStats(null, 1000);
    StatTrackingStore statTrackingStore = new StatTrackingStore(inMemoryEngine, null);
    QuotaLimitingStore quotaLimitingStore = new QuotaLimitingStore(statTrackingStore, statTrackingStore.getStats(), quotaStats, quotaStore, server.getMetadataStore());
    int targetRate = 50;
    // provide a quota of 100 gets/sec
    quotaStore.put(new ByteArray(QuotaUtils.makeQuotaKey(statTrackingStore.getName(), QuotaType.GET_THROUGHPUT).getBytes()), new Versioned<byte[]>("100.0".getBytes()), null);
    long testIntervalMs = 5000;
    long timeToSleepMs = 1000 / targetRate;
    long startMs = System.currentTimeMillis();
    ByteArray key = new ByteArray("some key".getBytes());
    while ((System.currentTimeMillis() - startMs) <= testIntervalMs) {
        quotaLimitingStore.get(key, null);
        Thread.sleep(timeToSleepMs);
    }
    assertEquals("No get operations should be throttled", 0, quotaStats.getRateLimitedGets());
    assertEquals("Put usage should be 0", 0, quotaStats.getQuotaPctUsedPut());
    assertEquals("delete usage should be 0", 0, quotaStats.getQuotaPctUsedDelete());
    assertEquals("getall usage should be 0", 0, quotaStats.getQuotaPctUsedGetAll());
    assertEquals("Computed usage pct must be close to actual observed qps", statTrackingStore.getStats().getThroughput(Tracked.GET), quotaStats.getQuotaPctUsedGet(), 1.0);
}
Also used : FileBackedCachingStorageEngine(voldemort.store.configuration.FileBackedCachingStorageEngine) StatTrackingStore(voldemort.store.stats.StatTrackingStore) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) ByteArray(voldemort.utils.ByteArray) QuotaLimitingStore(voldemort.store.quota.QuotaLimitingStore) QuotaLimitStats(voldemort.store.quota.QuotaLimitStats) File(java.io.File) Test(org.junit.Test)

Example 3 with QuotaLimitingStore

use of voldemort.store.quota.QuotaLimitingStore in project voldemort by voldemort.

the class StatusServlet method outputJSON.

protected void outputJSON(HttpServletResponse response) {
    StringBuilder sb = new StringBuilder("{\n");
    sb.append("  \"servertime\": \"");
    sb.append(new Date());
    sb.append("\",");
    sb.append("\n  \"server\": \"");
    sb.append(myMachine);
    sb.append("\",");
    sb.append("\n  \"node\": \"");
    sb.append(server.getMetadataStore().getNodeId());
    sb.append("\",");
    sb.append("\n  \"uptime\": \"");
    sb.append(abstractSocketService.getStatusManager().getFormattedUptime());
    sb.append("\",");
    sb.append("\n  \"num_workers\": ");
    sb.append(abstractSocketService.getStatusManager().getActiveWorkersCount());
    sb.append(",");
    sb.append("\n  \"pool_size\": ");
    sb.append(abstractSocketService.getStatusManager().getWorkerPoolSize());
    sb.append(",");
    sb.append("\n  \"stores\": {");
    int i = 0;
    for (Store<ByteArray, byte[], byte[]> store : server.getStoreRepository().getAllLocalStores()) {
        if (i++ > 0) {
            sb.append(",");
        }
        sb.append("\n    \"");
        sb.append(store.getName());
        sb.append("\" : {\n");
        if (store instanceof StatTrackingStore) {
            StatTrackingStore statStore = (StatTrackingStore) store;
            Map<Tracked, RequestCounter> stats = statStore.getStats().getCounters();
            for (Tracked t : Tracked.values()) {
                if (t == Tracked.EXCEPTION) {
                    continue;
                }
                sb.append(fillCommonStats(stats, t));
            }
            sb.append(",\n        \"num_exceptions\": ");
            sb.append(statStore.getStats().getCount(Tracked.EXCEPTION));
            sb.append("\n");
            sb.append("    }");
        }
        if (store instanceof QuotaLimitingStore) {
            QuotaLimitingStore quotaStore = (QuotaLimitingStore) store;
            Map<Tracked, RequestCounter> stats = quotaStore.getStats().getCounters();
            for (Tracked t : Tracked.values()) {
                if (t == Tracked.EXCEPTION) {
                    continue;
                }
                sb.append(fillCommonStats(stats, t));
            }
            sb.append(",\n        \"num_exceptions\": ");
            sb.append(quotaStore.getStats().getCount(Tracked.EXCEPTION));
            sb.append("\n");
            sb.append("    }");
        }
    }
    sb.append("\n  }\n");
    sb.append("}\n");
    try {
        response.setContentType("text/plain");
        OutputStreamWriter writer = new OutputStreamWriter(response.getOutputStream());
        writer.write(sb.toString());
        writer.flush();
    } catch (Exception e) {
        throw new VoldemortException(e);
    }
}
Also used : VoldemortException(voldemort.VoldemortException) Date(java.util.Date) ServletException(javax.servlet.ServletException) VoldemortException(voldemort.VoldemortException) IOException(java.io.IOException) UnknownHostException(java.net.UnknownHostException) StatTrackingStore(voldemort.store.stats.StatTrackingStore) Tracked(voldemort.store.stats.Tracked) RequestCounter(voldemort.store.stats.RequestCounter) ByteArray(voldemort.utils.ByteArray) QuotaLimitingStore(voldemort.store.quota.QuotaLimitingStore) OutputStreamWriter(java.io.OutputStreamWriter)

Aggregations

QuotaLimitingStore (voldemort.store.quota.QuotaLimitingStore)3 StatTrackingStore (voldemort.store.stats.StatTrackingStore)3 ByteArray (voldemort.utils.ByteArray)3 FileBackedCachingStorageEngine (voldemort.store.configuration.FileBackedCachingStorageEngine)2 QuotaLimitStats (voldemort.store.quota.QuotaLimitStats)2 File (java.io.File)1 IOException (java.io.IOException)1 OutputStreamWriter (java.io.OutputStreamWriter)1 UnknownHostException (java.net.UnknownHostException)1 Date (java.util.Date)1 MBeanServer (javax.management.MBeanServer)1 ObjectName (javax.management.ObjectName)1 ServletException (javax.servlet.ServletException)1 Test (org.junit.Test)1 VoldemortException (voldemort.VoldemortException)1 Cluster (voldemort.cluster.Cluster)1 InvalidMetadataCheckingStore (voldemort.store.invalidmetadata.InvalidMetadataCheckingStore)1 LoggingStore (voldemort.store.logging.LoggingStore)1 InMemoryStorageEngine (voldemort.store.memory.InMemoryStorageEngine)1 ProxyPutStats (voldemort.store.rebalancing.ProxyPutStats)1