Examples with InMemoryStorageEngine voldemort.store.memory.InMemoryStorageEngine used on opensource projects

Example 6 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class ServerTestUtils method createMetadataStore.

public static MetadataStore createMetadataStore(Cluster cluster, List<StoreDefinition> storeDefs, int nodeId) {
    Store<String, String, String> innerStore = new InMemoryStorageEngine<String, String, String>("inner-store");
    innerStore.put(MetadataStore.CLUSTER_KEY, new Versioned<String>(new ClusterMapper().writeCluster(cluster)), null);
    innerStore.put(MetadataStore.STORES_KEY, new Versioned<String>(new StoreDefinitionsMapper().writeStoreList(storeDefs)), null);
    return MetadataStore.createInMemoryMetadataStore(innerStore, nodeId);
}

Example 7 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class BlockingSlopPusherTest method setUp.

@Override
protected void setUp() throws Exception {
    Cluster cluster = makeCluster(3);
    repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
    MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
    Props props = new Props();
    props.put("node.id", 0);
    props.put("voldemort.home", "/");
    pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}

Example 8 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class QuotaLimitingStoreTest method testQuotaPctUsageCalculation.

@Test
public /**
     *  PS: Test will fail if for some reason we cannot do 50 ops/sec against a hash map. So yeah, pretty unlikely.
     */
void testQuotaPctUsageCalculation() throws Exception {
    File tempDir = TestUtils.createTempDir();
    FileBackedCachingStorageEngine quotaStore = new FileBackedCachingStorageEngine("quota-usage-test-store", tempDir.getAbsolutePath());
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("inMemoryBackingStore");
    QuotaLimitStats quotaStats = new QuotaLimitStats(null, 1000);
    StatTrackingStore statTrackingStore = new StatTrackingStore(inMemoryEngine, null);
    QuotaLimitingStore quotaLimitingStore = new QuotaLimitingStore(statTrackingStore, statTrackingStore.getStats(), quotaStats, quotaStore, server.getMetadataStore());
    int targetRate = 50;
    // provide a quota of 100 gets/sec
    quotaStore.put(new ByteArray(QuotaUtils.makeQuotaKey(statTrackingStore.getName(), QuotaType.GET_THROUGHPUT).getBytes()), new Versioned<byte[]>("100.0".getBytes()), null);
    long testIntervalMs = 5000;
    long timeToSleepMs = 1000 / targetRate;
    long startMs = System.currentTimeMillis();
    ByteArray key = new ByteArray("some key".getBytes());
    while ((System.currentTimeMillis() - startMs) <= testIntervalMs) {
        quotaLimitingStore.get(key, null);
        Thread.sleep(timeToSleepMs);
    }
    assertEquals("No get operations should be throttled", 0, quotaStats.getRateLimitedGets());
    assertEquals("Put usage should be 0", 0, quotaStats.getQuotaPctUsedPut());
    assertEquals("delete usage should be 0", 0, quotaStats.getQuotaPctUsedDelete());
    assertEquals("getall usage should be 0", 0, quotaStats.getQuotaPctUsedGetAll());
    assertEquals("Computed usage pct must be close to actual observed qps", statTrackingStore.getStats().getThroughput(Tracked.GET), quotaStats.getQuotaPctUsedGet(), 1.0);
}

Example 9 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class RoutedStoreTest method getStore.

private RoutedStore getStore(Cluster cluster, int reads, int writes, int threads, int failing, int sleepy, String strategy, VoldemortException e) throws Exception {
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int count = 0;
    for (Node n : cluster.getNodes()) {
        if (count >= cluster.getNumberOfNodes())
            throw new IllegalArgumentException(failing + " failing nodes, " + sleepy + " sleepy nodes, but only " + cluster.getNumberOfNodes() + " nodes in the cluster.");
        Store<ByteArray, byte[], byte[]> subStore = null;
        if (count < failing)
            subStore = new FailingStore<ByteArray, byte[], byte[]>("test", e);
        else if (count < failing + sleepy)
            subStore = new SleepyStore<ByteArray, byte[], byte[]>(Long.MAX_VALUE, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        else
            subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
        count += 1;
    }
    setFailureDetector(subStores);
    this.storeDef = ServerTestUtils.getStoreDef("test", reads + writes, reads, reads, writes, writes, strategy);
    routedStoreThreadPool = Executors.newFixedThreadPool(threads);
    RoutedStoreFactory routedStoreFactory = createFactory();
    return routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
}

Example 10 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class RoutedStoreTest method testPutDeleteZoneRouting.

@Test
public void testPutDeleteZoneRouting() throws Exception {
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 2);
    Versioned<byte[]> versioned = new Versioned<byte[]>(new byte[] { 1 });
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    Set<Integer> sleepy = Sets.newHashSet(4, 5, 6, 7);
    for (Node n : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> subStore = null;
        if (sleepy != null && sleepy.contains(n.getId()))
            subStore = new SleepyStore<ByteArray, byte[], byte[]>(SLEEPY_TIME, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        else
            subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
    }
    setFailureDetector(subStores);
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 1, 1, 1, 1, 0, 0, zoneReplicationFactor, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    routedStoreThreadPool = Executors.newFixedThreadPool(8);
    RoutedStoreFactory routedStoreFactory = createFactory();
    Store<ByteArray, byte[], byte[]> s1 = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(OPERATION_TIMEOUT));
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    List<Node> nodesRoutedTo = routingStrategy.routeRequest("test".getBytes());
    long start = System.nanoTime(), elapsed;
    try {
        s1.put(new ByteArray("test".getBytes()), versioned, null);
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).get(0), versioned);
    }
    // make sure the failure detector adds back any previously failed nodes
    Thread.sleep(BANNAGE_PERIOD + 100);
    start = System.nanoTime();
    try {
        s1.delete(new ByteArray("test".getBytes()), versioned.getVersion());
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).size(), 0);
    }
}