Examples with LoggingStore voldemort.store.logging.LoggingStore used on opensource projects

Example 1 with LoggingStore

use of voldemort.store.logging.LoggingStore in project voldemort by voldemort.

the class AbstractStoreClientFactory method getRawStore.

@SuppressWarnings("unchecked")
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver, String customStoresXml, String clusterXmlString, FailureDetector fd) {
    logger.info("Client zone-id [" + this.routedStoreConfig.getClientZoneId() + "] Attempting to get raw store [" + storeName + "] ");
    if (logger.isDebugEnabled()) {
        for (URI uri : bootstrapUrls) {
            logger.debug("Client Bootstrap url [" + uri + "]");
        }
    }
    // Get cluster and store metadata
    String clusterXml = clusterXmlString;
    if (clusterXml == null) {
        logger.debug("Fetching cluster.xml ...");
        clusterXml = bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY, bootstrapUrls);
    }
    this.cluster = clusterMapper.readCluster(new StringReader(clusterXml), false);
    String storesXml = customStoresXml;
    if (storesXml == null) {
        String storesKey = storeName;
        if (config.isFetchAllStoresXmlInBootstrap()) {
            storesKey = MetadataStore.STORES_KEY;
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Fetching store definition for Store " + storeName + " key " + storesKey);
        }
        storesXml = bootstrapMetadataWithRetries(storesKey, bootstrapUrls);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Obtained cluster metadata xml" + clusterXml);
        logger.debug("Obtained stores  metadata xml" + storesXml);
    }
    storeDefs = storeMapper.readStoreList(new StringReader(storesXml), false);
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null) {
        logger.error("Bootstrap - unknown store: " + storeName);
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug(this.cluster.toString(true));
        logger.debug(storeDef.toString());
    }
    boolean repairReads = !storeDef.isView();
    // construct mapping
    Map<Integer, Store<ByteArray, byte[], byte[]>> clientMapping = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = null;
    if (storeDef.hasHintedHandoffStrategyType())
        slopStores = Maps.newHashMap();
    for (Node node : this.cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> store = getStore(storeDef.getName(), node.getHost(), getPort(node), this.requestFormatType);
        clientMapping.put(node.getId(), store);
        NonblockingStore nonblockingStore = routedStoreFactory.toNonblockingStore(store);
        nonblockingStores.put(node.getId(), nonblockingStore);
        if (slopStores != null) {
            Store<ByteArray, byte[], byte[]> rawSlopStore = getStore("slop", node.getHost(), getPort(node), this.requestFormatType);
            Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(rawSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
            slopStores.put(node.getId(), slopStore);
            nonblockingSlopStores.put(node.getId(), routedStoreFactory.toNonblockingStore(rawSlopStore));
        }
    }
    /*
         * Check if we need to retrieve a reference to the failure detector. For
         * system stores - the FD reference would be passed in.
         */
    FailureDetector failureDetectorRef = fd;
    if (failureDetectorRef == null) {
        failureDetectorRef = getFailureDetector();
    } else {
        logger.debug("Using existing failure detector.");
    }
    this.routedStoreConfig.setRepairReads(repairReads);
    Store<ByteArray, byte[], byte[]> store = routedStoreFactory.create(this.cluster, storeDef, clientMapping, nonblockingStores, slopStores, nonblockingSlopStores, failureDetectorRef, this.routedStoreConfig);
    store = new LoggingStore(store);
    if (isJmxEnabled) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.aggregateStats, this.cachedStoreStats);
        statStore.getStats().registerJmx(identifierString);
        store = statStore;
    }
    if (this.config.isEnableCompressionLayer()) {
        if (storeDef.getKeySerializer().hasCompression() || storeDef.getValueSerializer().hasCompression()) {
            store = new CompressingStore(store, getCompressionStrategy(storeDef.getKeySerializer()), getCompressionStrategy(storeDef.getValueSerializer()));
        }
    }
    /*
         * Initialize the finalstore object only once the store object itself is
         * wrapped by a StatrackingStore seems like the finalstore object is
         * redundant?
         */
    Store<K, V, T> finalStore = (Store<K, V, T>) store;
    if (this.config.isEnableSerializationLayer()) {
        Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(storeDef.getKeySerializer());
        Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(storeDef.getValueSerializer());
        if (storeDef.isView() && (storeDef.getTransformsSerializer() == null))
            throw new SerializationException("Transforms serializer must be specified with a view ");
        Serializer<T> transformsSerializer = (Serializer<T>) serializerFactory.getSerializer(storeDef.getTransformsSerializer() != null ? storeDef.getTransformsSerializer() : new SerializerDefinition("identity"));
        finalStore = SerializingStore.wrap(store, keySerializer, valueSerializer, transformsSerializer);
    }
    // resolver (if they gave us one)
    if (this.config.isEnableInconsistencyResolvingLayer()) {
        InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver() : resolver;
        finalStore = new InconsistencyResolvingStore<K, V, T>(finalStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    }
    return finalStore;
}

Example 2 with LoggingStore

use of voldemort.store.logging.LoggingStore in project voldemort by voldemort.

the class StorageService method registerEngine.

/**
 * Register the given engine with the storage repository
 *
 * @param engine Register the storage engine
 * @param isReadOnly Boolean indicating if this store is read-only
 * @param storeType The type of the store
 * @param storeDef store definition for the store to be registered
 */
public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, StoreDefinition storeDef) {
    Cluster cluster = this.metadata.getCluster();
    storeRepository.addStorageEngine(engine);
    /* Now add any store wrappers that are enabled */
    Store<ByteArray, byte[], byte[]> store = engine;
    boolean isMetadata = store.getName().compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
    boolean isSlop = storeType.compareTo("slop") == 0;
    boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
    if (voldemortConfig.isVerboseLoggingEnabled())
        store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
    if (!isSlop) {
        if (!isReadOnly && !isMetadata && !isView) {
            // wrap store to enforce retention policy
            if (voldemortConfig.isEnforceRetentionPolicyOnRead() && storeDef != null) {
                RetentionEnforcingStore retentionEnforcingStore = new RetentionEnforcingStore(store, storeDef, voldemortConfig.isDeleteExpiredValuesOnRead(), SystemTime.INSTANCE);
                metadata.addMetadataStoreListener(store.getName(), retentionEnforcingStore);
                store = retentionEnforcingStore;
            }
            if (voldemortConfig.isEnableRebalanceService()) {
                ProxyPutStats proxyPutStats = new ProxyPutStats(aggregatedProxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    JmxUtils.registerMbean(proxyPutStats, JmxUtils.createObjectName("voldemort.store.rebalancing", engine.getName() + "-proxy-puts"));
                }
                store = new RedirectingStore(store, metadata, storeRepository, failureDetector, storeFactory, proxyPutWorkerPool, proxyPutStats);
                if (voldemortConfig.isJmxEnabled()) {
                    MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
                    ObjectName name = null;
                    if (this.voldemortConfig.isEnableJmxClusterName())
                        name = JmxUtils.createObjectName(cluster.getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    else
                        name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
                    synchronized (mbeanServer) {
                        if (mbeanServer.isRegistered(name))
                            JmxUtils.unregisterMbean(mbeanServer, name);
                        JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(store), name);
                    }
                }
            }
        }
        if (voldemortConfig.isMetadataCheckingEnabled() && !isMetadata) {
            store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
        }
    }
    if (voldemortConfig.isStatTrackingEnabled()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
        store = statStore;
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName name = null;
            if (this.voldemortConfig.isEnableJmxClusterName())
                name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
            else
                name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
            synchronized (mbeanServer) {
                if (mbeanServer.isRegistered(name))
                    JmxUtils.unregisterMbean(mbeanServer, name);
                JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
            }
        }
        // metadata store)
        if (voldemortConfig.isEnableQuotaLimiting() && !isMetadata) {
            StoreStats currentStoreStats = statStore.getStats();
            FileBackedCachingStorageEngine quotaStore = (FileBackedCachingStorageEngine) storeRepository.getStorageEngine(SystemStoreConstants.SystemStoreName.voldsys$_store_quotas.toString());
            QuotaLimitStats quotaStats = new QuotaLimitStats(this.aggregatedQuotaStats);
            QuotaLimitingStore rateLimitingStore = new QuotaLimitingStore(store, currentStoreStats, quotaStats, quotaStore, metadata);
            if (voldemortConfig.isJmxEnabled()) {
                JmxUtils.registerMbean(quotaStats, JmxUtils.createObjectName("voldemort.store.quota", store.getName() + "-quota-limit-stats"));
            }
            store = rateLimitingStore;
        }
    }
    storeRepository.addLocalStore(store);
}

Example 3 with LoggingStore

use of voldemort.store.logging.LoggingStore in project voldemort by voldemort.

the class StorageService method registerSystemEngine.

public void registerSystemEngine(StorageEngine<ByteArray, byte[], byte[]> engine) {
    Cluster cluster = this.metadata.getCluster();
    storeRepository.addStorageEngine(engine);
    /* Now add any store wrappers that are enabled */
    Store<ByteArray, byte[], byte[]> store = engine;
    if (voldemortConfig.isVerboseLoggingEnabled())
        store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
    if (voldemortConfig.isMetadataCheckingEnabled())
        store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
    if (voldemortConfig.isStatTrackingEnabled()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
        store = statStore;
        if (voldemortConfig.isJmxEnabled()) {
            MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
            ObjectName name = null;
            if (this.voldemortConfig.isEnableJmxClusterName())
                name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
            else
                name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
            synchronized (mbeanServer) {
                if (mbeanServer.isRegistered(name))
                    JmxUtils.unregisterMbean(mbeanServer, name);
                JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
            }
        }
    }
    storeRepository.addLocalStore(store);
}

Example 4 with LoggingStore

use of voldemort.store.logging.LoggingStore in project voldemort by voldemort.

the class HintedHandoffFailureTest method customSetup.

/**
 * Setup a cluster with 3 nodes, with the following characteristics:
 *
 * If FAILURE_MODE is FAIL_FIRST_REPLICA_NODE set the first replica store to
 * a sleepy force failing store
 *
 * If FAILURE_MODE is FAIL_ALL_REPLICAS: set all replicas to sleepy force
 * failing store
 *
 * Pseudo master : Standard In-memory store (wrapped by Logging store)
 *
 * In memory slop stores
 *
 * @param key The ByteArray representation of the key
 * @param failureMode The Failure mode for the replicas
 *
 * @throws Exception
 */
public List<Integer> customSetup(ByteArray key, FAILURE_MODE failureMode, ReplicaFactor replicaFactor, long sleepTime, long hintDelayTimeMs) throws Exception {
    cluster = getThreeNodeCluster();
    storeDef = getStoreDef(STORE_NAME, replicaFactor, RoutingStrategyType.CONSISTENT_STRATEGY);
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryStorageEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
    LoggingStore<ByteArray, byte[], byte[]> loggingStore = new LoggingStore<ByteArray, byte[], byte[]>(inMemoryStorageEngine);
    VoldemortException e = new UnreachableStoreException("Node down");
    ForceFailStore<ByteArray, byte[], byte[]> failureStore = new ForceFailStore<ByteArray, byte[], byte[]>(loggingStore, e);
    SleepyStore<ByteArray, byte[], byte[]> sleepyFailureStore = new SleepyStore<ByteArray, byte[], byte[]>(sleepTime, failureStore);
    failureStore.setFail(true);
    List<Integer> failingNodeIdList = Lists.newArrayList();
    List<Node> replicaList = strategy.routeRequest(key.get());
    switch(failureMode) {
        case FAIL_FIRST_REPLICA_NODE:
            failingNodeIdList.add(replicaList.get(1).getId());
            break;
        case FAIL_ALL_REPLICAS:
            for (int nodeId = 1; nodeId < replicaList.size(); nodeId++) {
                failingNodeIdList.add(nodeId);
            }
            break;
    }
    subStores.clear();
    for (int i = 0; i < NUM_NODES_TOTAL; i++) {
        if (failingNodeIdList.contains(i)) {
            subStores.put(i, sleepyFailureStore);
        } else {
            subStores.put(i, loggingStore);
        }
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(NUM_THREADS);
    routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    for (Node node : cluster.getNodes()) {
        int nodeId = node.getId();
        SlopStorageEngine slopStorageEngine = new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(SLOP_STORE_NAME), cluster);
        StorageEngine<ByteArray, Slop, byte[]> storageEngine = slopStorageEngine.asSlopStore();
        nonblockingSlopStores.put(nodeId, routedStoreFactory.toNonblockingStore(slopStorageEngine));
        slopStores.put(nodeId, storageEngine);
    }
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : subStores.entrySet()) nonblockingStores.put(entry.getKey(), routedStoreFactory.toNonblockingStore(entry.getValue()));
    store = new DelayedPutPipelineRoutedStore(subStores, nonblockingStores, slopStores, nonblockingSlopStores, cluster, storeDef, failureDetector, hintDelayTimeMs);
    return failingNodeIdList;
}