Examples with StoreStatsJmx voldemort.store.stats.StoreStatsJmx used on opensource projects

Example 1 with StoreStatsJmx

use of voldemort.store.stats.StoreStatsJmx in project voldemort by voldemort.

the class RESTClientFactory method getRawStore.

@Override
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver) {
    Store<K, V, T> clientStore = null;
    // The lowest layer : Transporting request to coordinator
    R2Store r2store = null;
    this.d2Client = restClientFactoryConfig.getD2Client();
    if (this.d2Client == null) {
        logger.info("Using transportclient since d2client is not available");
        Map<String, String> properties = new HashMap<String, String>();
        properties.put(HttpClientFactory.HTTP_POOL_SIZE, Integer.toString(this.config.getMaxR2ConnectionPoolSize()));
        transportClient = _clientFactory.getClient(properties);
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.transportClient, this.config);
    } else {
        logger.info("Using d2client");
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.d2Client, this.config);
    }
    this.rawStoreList.add(r2store);
    // bootstrap from the coordinator and obtain all the serialization
    // information.
    String serializerInfoXml = r2store.getSerializerInfoXml();
    SerializerDefinition keySerializerDefinition = RestUtils.parseKeySerializerDefinition(serializerInfoXml);
    SerializerDefinition valueSerializerDefinition = RestUtils.parseValueSerializerDefinition(serializerInfoXml);
    synchronized (this) {
        keySerializerMap.put(storeName, keySerializerDefinition);
        valueSerializerMap.put(storeName, valueSerializerDefinition);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Bootstrapping for " + storeName + ": Key serializer " + keySerializerDefinition);
        logger.debug("Bootstrapping for " + storeName + ": Value serializer " + valueSerializerDefinition);
    }
    // Start building the stack..
    // First, the transport layer
    Store<ByteArray, byte[], byte[]> store = r2store;
    // TODO: Add identifierString to the Mbean name
    if (this.config.isEnableJmx()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.stats);
        store = statStore;
        JmxUtils.registerMbean(new StoreStatsJmx(statStore.getStats()), JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName()));
    }
    // Add compression layer
    if (keySerializerDefinition.hasCompression() || valueSerializerDefinition.hasCompression()) {
        store = new CompressingStore(store, new CompressionStrategyFactory().get(keySerializerDefinition.getCompression()), new CompressionStrategyFactory().get(valueSerializerDefinition.getCompression()));
    }
    // Add Serialization layer
    Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(keySerializerDefinition);
    Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(valueSerializerDefinition);
    clientStore = SerializingStore.wrap(store, keySerializer, valueSerializer, null);
    // Add inconsistency Resolving layer
    InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver<V>() : resolver;
    clientStore = new InconsistencyResolvingStore<K, V, T>(clientStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver<V>(), secondaryResolver));
    return clientStore;
}

Example 2 with StoreStatsJmx

use of voldemort.store.stats.StoreStatsJmx in project voldemort by voldemort.

the class StorageService method startInner.

@Override
protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");
    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations()) initStorageConfig(configClassName);
    /* Initialize view storage configuration */
    storageConfigs.put(ViewStorageConfiguration.TYPE_NAME, new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));
    /* Initialize system stores */
    initSystemStores();
    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {
        logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
        StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
        if (config == null)
            throw new ConfigurationException("Attempt to open store " + SlopStorageEngine.SLOP_STORE_NAME + " but " + voldemortConfig.getSlopStoreType() + " storage engine has not been enabled.");
        // make a dummy store definition object
        StoreDefinition slopStoreDefinition = new StoreDefinition(SlopStorageEngine.SLOP_STORE_NAME, null, null, null, null, null, null, RoutingStrategyType.CONSISTENT_STRATEGY, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        SlopStorageEngine slopEngine = new SlopStorageEngine(config.getStore(slopStoreDefinition, new RoutingStrategyFactory().updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())), metadata.getCluster());
        registerInternalEngine(slopEngine, false, "slop");
        storeRepository.setSlopStore(slopEngine);
        if (voldemortConfig.isSlopPusherJobEnabled()) {
            // Now initialize the pusher job after some time
            GregorianCalendar cal = new GregorianCalendar();
            cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
            Date nextRun = cal.getTime();
            logger.info("Initializing slop pusher job type " + voldemortConfig.getPusherType() + " at " + nextRun);
            scheduler.schedule("slop", (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0) ? new BlockingSlopPusherJob(storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper) : new StreamingSlopPusherJob(storeRepository, metadata, slopStreamingFailureDetector, voldemortConfig, scanPermitWrapper), nextRun, voldemortConfig.getSlopFrequencyMs());
        }
        // Create a SlopPurgeJob object and register it
        if (voldemortConfig.isSlopPurgeJobEnabled()) {
            logger.info("Initializing Slop Purge job");
            SlopPurgeJob job = new SlopPurgeJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
            JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
            storeRepository.registerSlopPurgeJob(job);
        }
    }
    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
        logger.info("Initializing repair job.");
        RepairJob job = new RepairJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getRepairJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerRepairJob(job);
    }
    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
        logger.info("Intializing prune job");
        VersionedPutPruneJob job = new VersionedPutPruneJob(storeRepository, metadata, scanPermitWrapper, voldemortConfig.getPruneJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerPruneJob(job);
    }
    List<StoreDefinition> storeDefs = new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");
    logger.info("Validating schemas:");
    StoreDefinitionUtils.validateSchemasAsNeeded(storeDefs);
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView())
        openStore(def);
    // those stores
    for (StoreDefinition def : storeDefs) {
        if (def.isView())
            openStore(def);
    }
    initializeMetadataVersions(storeDefs);
    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
        if (this.voldemortConfig.isEnableJmxClusterName())
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName(metadata.getCluster().getName() + ".voldemort.store.stats.aggregate", "aggregate-perf"));
        else
            JmxUtils.registerMbean(new StoreStatsJmx(this.storeStats), JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));
    List<StorageEngine> listOfDisabledStores = Lists.newArrayList();
    for (StorageEngine storageEngine : storeRepository.getAllStorageEngines()) {
        try {
            StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
            if (storeVersionManager.hasAnyDisabledVersion()) {
                listOfDisabledStores.add(storageEngine);
                logger.warn("The following store is marked as disabled: " + storageEngine.getName());
            // Must put server in offline mode.
            }
        } catch (NoSuchCapabilityException e) {
        // Not a read-only store: no-op
        }
    }
    if (listOfDisabledStores.isEmpty()) {
        logger.info("All stores initialized.");
    } else {
        throw new DisabledStoreException("All stores initialized, but the server needs to go " + "in offline mode because some store(s) are disabled.");
    }
}

Example 3 with StoreStatsJmx

use of voldemort.store.stats.StoreStatsJmx in project voldemort by voldemort.

the class StorageService method registerEngine.

/**
 * Register the given engine with the storage repository
 *
 * @param engine Register the storage engine
 * @param isReadOnly Boolean indicating if this store is read-only
 * @param storeType The type of the store
 * @param storeDef store definition for the store to be registered
 */
public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, StoreDefinition storeDef) {
    Cluster cluster = this.metadata.getCluster();
    storeRepository.addStorageEngine(engine);
    /* Now add any store wrappers that are enabled */
    Store<ByteArray, byte[], byte[]> store = engine;
    boolean isMetadata = store.getName().compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
    boolean isSlop = storeType.compareTo("slop") == 0;
    boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
    if (voldemortConfig.isVerboseLoggingEnabled())
        store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
    if (!isSlop) {
        if (!isReadOnly && !isMetadata && !isView) {
            // wrap store to enforce retention policy
            if (voldemortConfig.isEnforceRetentionPolicyOnRead() && storeDef != null) {
                RetentionEnforcingStore retentionEnforcingStore = new RetentionEnforcingStore(store, storeDef, voldemortConfig.isDeleteExpiredValuesOnRead(), SystemTime.INSTANCE);
                metadata.addMetadataStoreListener(store.getName(), retentionEnforcingStore);
                store = retentionEnforcingStore;
            }
            if (voldemortConfig.isEnableRebalanceService()) {
                ProxyPutStats proxyPutStats = new ProxyPutStats(aggregatedProxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    JmxUtils.registerMbean(proxyPutStats, JmxUtils.createObjectName("voldemort.store.rebalancing", engine.getName() + "-proxy-puts"));
                }
                store = new RedirectingStore(store, metadata, storeRepository, failureDetector, storeFactory, proxyPutWorkerPool, proxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
                    ObjectName name = null;
                    if (this.voldemortConfig.isEnableJmxClusterName())
                        name = JmxUtils.createObjectName(cluster.getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    else
                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    synchronized (mbeanServer) {
                        if (mbeanServer.isRegistered(name))
                            JmxUtils.unregisterMbean(mbeanServer, name);
                        JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(store), name);
                    }
                }
            }
        }
        if (voldemortConfig.isMetadataCheckingEnabled() && !isMetadata) {
            store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
        }
    }
    if (voldemortConfig.isStatTrackingEnabled()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
        store = statStore;
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName name = null;
            if (this.voldemortConfig.isEnableJmxClusterName())
                name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
            else
                name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
            synchronized (mbeanServer) {
                if (mbeanServer.isRegistered(name))
                    JmxUtils.unregisterMbean(mbeanServer, name);
                JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
            }
        }
        // metadata store)
        if (voldemortConfig.isEnableQuotaLimiting() && !isMetadata) {
            StoreStats currentStoreStats = statStore.getStats();
            FileBackedCachingStorageEngine quotaStore = (FileBackedCachingStorageEngine) storeRepository.getStorageEngine(SystemStoreConstants.SystemStoreName.voldsys$_store_quotas.toString());
            QuotaLimitStats quotaStats = new QuotaLimitStats(this.aggregatedQuotaStats);
            QuotaLimitingStore rateLimitingStore = new QuotaLimitingStore(store, currentStoreStats, quotaStats, quotaStore, metadata);
            if (voldemortConfig.isJmxEnabled()) {
                JmxUtils.registerMbean(quotaStats, JmxUtils.createObjectName("voldemort.store.quota", store.getName() + "-quota-limit-stats"));
            }
            store = rateLimitingStore;
        }
    }
    storeRepository.addLocalStore(store);
}

Example 4 with StoreStatsJmx

use of voldemort.store.stats.StoreStatsJmx in project voldemort by voldemort.

the class RestPipelineFactory method createAndRegisterMBeansForAllStoreStats.

public void createAndRegisterMBeansForAllStoreStats(VoldemortConfig config, List<StoreDefinition> storeDefinitions) {
    storeStatsMap = new ConcurrentHashMap<String, StoreStats>();
    boolean isJmxEnabled = config.isJmxEnabled();
    for (StoreDefinition storeDefinition : Utils.notNull(storeDefinitions)) {
        String storeName = storeDefinition.getName();
        // Add to concurrentHashMap
        storeStatsMap.put(storeName, new StoreStats(storeName, aggregatedStoreStats));
        // Register storeStats MBeans for every store
        if (isJmxEnabled) {
            JmxUtils.registerMbean(new StoreStatsJmx(storeStatsMap.get(storeName)), JmxUtils.createObjectName(JmxUtils.getPackageName(this.getClass()) + ".store.stats", storeName));
        }
    }
    // Register Mbean for aggregated server store stats
    if (isJmxEnabled) {
        JmxUtils.registerMbean(new StoreStatsJmx(aggregatedStoreStats), JmxUtils.createObjectName(JmxUtils.getPackageName(this.getClass()) + ".store.stats", JmxUtils.getClassName(this.aggregatedStoreStats.getClass())));
    }
}

Example 5 with StoreStatsJmx

use of voldemort.store.stats.StoreStatsJmx in project voldemort by voldemort.

the class StorageWorkerThread method run.

@Override
public void run() {
    Object message = messageEvent.getMessage();
    boolean fromLocalZone = false;
    if (message instanceof VoldemortStoreRequest) {
        VoldemortStoreRequest storeRequestObject = (VoldemortStoreRequest) message;
        this.requestObject = storeRequestObject.getRequestObject();
        long now = System.currentTimeMillis();
        if (requestObject.getRequestOriginTimeInMs() + requestObject.getRoutingTimeoutInMs() <= now) {
            RestErrorHandler.writeErrorResponse(messageEvent, HttpResponseStatus.REQUEST_TIMEOUT, "current time: " + now + "\torigin time: " + requestObject.getRequestOriginTimeInMs() + "\ttimeout in ms: " + requestObject.getRoutingTimeoutInMs());
            return;
        } else {
            fromLocalZone = isLocalZoneId(storeRequestObject.getZoneId());
            this.store = storeRequestObject.getStore();
            String storeName = store.getName();
            performanceStats = storeStatsMap.get(storeName);
            if (performanceStats == null) {
                // Add to concurrentHashMap
                storeStatsMap.putIfAbsent(storeName, new StoreStats(storeName, aggregatedStoreStats));
                performanceStats = storeStatsMap.get(storeName);
                // Register MBeans for new store performance stats
                if (isJmxEnabled) {
                    JmxUtils.registerMbean(new StoreStatsJmx(performanceStats), JmxUtils.createObjectName(JmxUtils.getPackageName(this.getClass()) + ".store.stats", storeName));
                }
            }
            switch(requestObject.getOperationType()) {
                case VoldemortOpCode.GET_OP_CODE:
                    if (logger.isDebugEnabled()) {
                        logger.debug("Incoming get request");
                    }
                    try {
                        List<Versioned<byte[]>> versionedValues = store.get(requestObject.getKey(), null);
                        // handle non existing key
                        if (versionedValues.size() > 0) {
                            GetResponseSender responseConstructor = new GetResponseSender(messageEvent, requestObject.getKey(), versionedValues, storeName);
                            responseConstructor.sendResponse(performanceStats, fromLocalZone, requestObject.getRequestOriginTimeInMs());
                        } else {
                            logger.error("Error when doing get. Key does not exist");
                            RestErrorHandler.writeErrorResponse(messageEvent, NOT_FOUND, "Key does not exist");
                        }
                    } catch (Exception e) {
                        getErrorHandler.handleExceptions(messageEvent, e);
                    }
                    break;
                case VoldemortOpCode.GET_ALL_OP_CODE:
                    if (logger.isDebugEnabled()) {
                        logger.debug("Incoming get all request");
                    }
                    try {
                        Map<ByteArray, List<Versioned<byte[]>>> keyValuesMap = store.getAll(requestObject.getIterableKeys(), null);
                        // check if there is at least one valid key
                        // before sending response
                        boolean hasAtleastOneValidKey = false;
                        for (List<Versioned<byte[]>> values : keyValuesMap.values()) {
                            if (values.size() > 0) {
                                hasAtleastOneValidKey = true;
                                break;
                            }
                        }
                        if (hasAtleastOneValidKey) {
                            GetAllResponseSender responseConstructor = new GetAllResponseSender(messageEvent, keyValuesMap, storeName);
                            responseConstructor.sendResponse(performanceStats, fromLocalZone, requestObject.getRequestOriginTimeInMs());
                        } else {
                            logger.error("Error when doing getall. Key does not exist or key is null");
                            RestErrorHandler.writeErrorResponse(messageEvent, NOT_FOUND, "Key does not exist or key is null");
                        }
                    } catch (Exception e) {
                        getErrorHandler.handleExceptions(messageEvent, e);
                    }
                    break;
                case VoldemortOpCode.PUT_OP_CODE:
                    if (logger.isDebugEnabled()) {
                        logger.debug("Incoming put request");
                    }
                    try {
                        store.put(requestObject.getKey(), requestObject.getValue(), null);
                        PutResponseSender responseConstructor = new PutResponseSender(messageEvent);
                        responseConstructor.sendResponse(performanceStats, fromLocalZone, requestObject.getRequestOriginTimeInMs());
                    } catch (Exception e) {
                        putErrorHandler.handleExceptions(messageEvent, e);
                    }
                    break;
                case VoldemortOpCode.DELETE_OP_CODE:
                    if (logger.isDebugEnabled()) {
                        logger.debug("Incoming delete request");
                    }
                    try {
                        boolean result = store.delete(requestObject.getKey(), requestObject.getVersion());
                        if (!result) {
                            logger.error("Error when doing delete. Non Existing key/version. Nothing to delete");
                            RestErrorHandler.writeErrorResponse(messageEvent, NOT_FOUND, "Non Existing key/version. Nothing to delete");
                            break;
                        }
                        DeleteResponseSender responseConstructor = new DeleteResponseSender(messageEvent);
                        responseConstructor.sendResponse(performanceStats, fromLocalZone, requestObject.getRequestOriginTimeInMs());
                    } catch (Exception e) {
                        deleteErrorHandler.handleExceptions(messageEvent, e);
                    }
                    break;
                case VoldemortOpCode.GET_VERSION_OP_CODE:
                    if (logger.isDebugEnabled()) {
                        logger.debug("Incoming get version request");
                    }
                    try {
                        List<Version> versions = store.getVersions(requestObject.getKey());
                        // handle non existing key
                        if (versions.size() > 0) {
                            GetVersionResponseSender responseConstructor = new GetVersionResponseSender(messageEvent, requestObject.getKey(), versions, storeName);
                            responseConstructor.sendResponse(performanceStats, fromLocalZone, requestObject.getRequestOriginTimeInMs());
                        } else {
                            logger.error("Error when doing getversion. Key does not exist or key is null");
                            RestErrorHandler.writeErrorResponse(messageEvent, NOT_FOUND, "Key does not exist or key is null");
                        }
                    } catch (Exception e) {
                        getVersionErrorHandler.handleExceptions(messageEvent, e);
                    }
                    break;
                default:
                    // above, the code stops here.
                    return;
            }
        }
    }
}