use of voldemort.store.retention.RetentionEnforcingStore in project voldemort by voldemort.
the class StorageService method registerEngine.
/**
* Register the given engine with the storage repository
*
* @param engine Register the storage engine
* @param isReadOnly Boolean indicating if this store is read-only
* @param storeType The type of the store
* @param storeDef store definition for the store to be registered
*/
public void registerEngine(StorageEngine<ByteArray, byte[], byte[]> engine, boolean isReadOnly, String storeType, StoreDefinition storeDef) {
Cluster cluster = this.metadata.getCluster();
storeRepository.addStorageEngine(engine);
/* Now add any store wrappers that are enabled */
Store<ByteArray, byte[], byte[]> store = engine;
boolean isMetadata = store.getName().compareTo(MetadataStore.METADATA_STORE_NAME) == 0;
boolean isSlop = storeType.compareTo("slop") == 0;
boolean isView = storeType.compareTo(ViewStorageConfiguration.TYPE_NAME) == 0;
if (voldemortConfig.isVerboseLoggingEnabled())
store = new LoggingStore<ByteArray, byte[], byte[]>(store, cluster.getName(), SystemTime.INSTANCE);
if (!isSlop) {
if (!isReadOnly && !isMetadata && !isView) {
// wrap store to enforce retention policy
if (voldemortConfig.isEnforceRetentionPolicyOnRead() && storeDef != null) {
RetentionEnforcingStore retentionEnforcingStore = new RetentionEnforcingStore(store, storeDef, voldemortConfig.isDeleteExpiredValuesOnRead(), SystemTime.INSTANCE);
metadata.addMetadataStoreListener(store.getName(), retentionEnforcingStore);
store = retentionEnforcingStore;
}
if (voldemortConfig.isEnableRebalanceService()) {
ProxyPutStats proxyPutStats = new ProxyPutStats(aggregatedProxyPutStats);
if (voldemortConfig.isJmxEnabled()) {
JmxUtils.registerMbean(proxyPutStats, JmxUtils.createObjectName("voldemort.store.rebalancing", engine.getName() + "-proxy-puts"));
}
store = new RedirectingStore(store, metadata, storeRepository, failureDetector, storeFactory, proxyPutWorkerPool, proxyPutStats);
if (voldemortConfig.isJmxEnabled()) {
MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
ObjectName name = null;
if (this.voldemortConfig.isEnableJmxClusterName())
name = JmxUtils.createObjectName(cluster.getName() + "." + JmxUtils.getPackageName(RedirectingStore.class), store.getName());
else
name = JmxUtils.createObjectName(JmxUtils.getPackageName(RedirectingStore.class), store.getName());
synchronized (mbeanServer) {
if (mbeanServer.isRegistered(name))
JmxUtils.unregisterMbean(mbeanServer, name);
JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(store), name);
}
}
}
}
if (voldemortConfig.isMetadataCheckingEnabled() && !isMetadata) {
store = new InvalidMetadataCheckingStore(metadata.getNodeId(), store, metadata);
}
}
if (voldemortConfig.isStatTrackingEnabled()) {
StatTrackingStore statStore = new StatTrackingStore(store, this.storeStats);
store = statStore;
if (voldemortConfig.isJmxEnabled()) {
MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
ObjectName name = null;
if (this.voldemortConfig.isEnableJmxClusterName())
name = JmxUtils.createObjectName(metadata.getCluster().getName() + "." + JmxUtils.getPackageName(store.getClass()), store.getName());
else
name = JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName());
synchronized (mbeanServer) {
if (mbeanServer.isRegistered(name))
JmxUtils.unregisterMbean(mbeanServer, name);
JmxUtils.registerMbean(mbeanServer, JmxUtils.createModelMBean(new StoreStatsJmx(statStore.getStats())), name);
}
}
// metadata store)
if (voldemortConfig.isEnableQuotaLimiting() && !isMetadata) {
StoreStats currentStoreStats = statStore.getStats();
FileBackedCachingStorageEngine quotaStore = (FileBackedCachingStorageEngine) storeRepository.getStorageEngine(SystemStoreConstants.SystemStoreName.voldsys$_store_quotas.toString());
QuotaLimitStats quotaStats = new QuotaLimitStats(this.aggregatedQuotaStats);
QuotaLimitingStore rateLimitingStore = new QuotaLimitingStore(store, currentStoreStats, quotaStats, quotaStore, metadata);
if (voldemortConfig.isJmxEnabled()) {
JmxUtils.registerMbean(quotaStats, JmxUtils.createObjectName("voldemort.store.quota", store.getName() + "-quota-limit-stats"));
}
store = rateLimitingStore;
}
}
storeRepository.addLocalStore(store);
}
use of voldemort.store.retention.RetentionEnforcingStore in project voldemort by voldemort.
the class DataCleanupJobTest method runRetentionEnforcingStoreTest.
private void runRetentionEnforcingStoreTest(boolean onlineDeletes) throws InterruptedException {
time.setTime(System.currentTimeMillis());
StoreDefinition retentionStoreDef = new StoreDefinitionsMapper().readStoreList(new StringReader(VoldemortTestConstants.getStoreDefinitionsWithRetentionXml())).get(0);
RetentionEnforcingStore store = new RetentionEnforcingStore(engine, retentionStoreDef, onlineDeletes, time);
// do a bunch of puts
store.put(new ByteArray("k1".getBytes()), new Versioned<byte[]>("v1".getBytes()), null);
store.put(new ByteArray("k2".getBytes()), new Versioned<byte[]>("v2".getBytes()), null);
long writeMs = System.currentTimeMillis();
// wait for a bit and then do more puts
Thread.sleep(2000);
store.put(new ByteArray("k3".getBytes()), new Versioned<byte[]>("v3".getBytes()), null);
store.put(new ByteArray("k4".getBytes()), new Versioned<byte[]>("v4".getBytes()), null);
// move time forward just enough such that some keys will have expired.
time.setTime(writeMs + retentionStoreDef.getRetentionDays() * Time.MS_PER_DAY + 1);
assertEquals("k1 should have expired", 0, store.get(new ByteArray("k1".getBytes()), null).size());
assertEquals("k2 should have expired", 0, store.get(new ByteArray("k2".getBytes()), null).size());
assertTrue("k3 should not have expired", store.get(new ByteArray("k3".getBytes()), null).size() > 0);
assertTrue("k4 should not have expired", store.get(new ByteArray("k4".getBytes()), null).size() > 0);
// get all with k1, k4 should return a map with k4 alone
Map<ByteArray, List<Versioned<byte[]>>> getAllResult = store.getAll(Arrays.asList(new ByteArray("k1".getBytes()), new ByteArray("k4".getBytes())), null);
assertEquals("map should contain one element only", 1, getAllResult.size());
assertEquals("k1 should not be present", false, getAllResult.containsKey(new ByteArray("k1".getBytes())));
assertEquals("k4 should be present", true, getAllResult.containsKey(new ByteArray("k4".getBytes())));
// if online deletes are not configured, we should see the deleted keys
// in the base bdb store, so the datacleanup job can go and delete them
assertEquals("k1 should be present", !onlineDeletes, engine.get(new ByteArray("k1".getBytes()), null).size() > 0);
assertEquals("k2 should be present", !onlineDeletes, engine.get(new ByteArray("k2".getBytes()), null).size() > 0);
// delete everything for next run
engine.truncate();
}
Aggregations