Examples with InMemoryStorageEngine voldemort.store.memory.InMemoryStorageEngine used on opensource projects

Example 11 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class GetallNodeReachTest method makeStore.

private void makeStore() {
    subStores = Maps.newHashMap();
    for (Node n : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
    }
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(Executors.newFixedThreadPool(2));
    store = routedStoreFactory.create(cluster, storeDef, subStores, new NoopFailureDetector(), new RoutedStoreConfig().setTimeoutConfig(new TimeoutConfig(1000L)));
}

Example 12 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class RemoteStoreComparisonTest method main.

public static void main(String[] args) throws Exception {
    if (args.length != 2)
        Utils.croak("USAGE: java " + RemoteStoreComparisonTest.class.getName() + " numRequests numThreads [useNio]");
    int numRequests = Integer.parseInt(args[0]);
    int numThreads = Integer.parseInt(args[1]);
    boolean useNio = args.length > 2 ? args[2].equals("true") : false;
    /** * In memory test ** */
    final Store<byte[], byte[], byte[]> memStore = new InMemoryStorageEngine<byte[], byte[], byte[]>("test");
    PerformanceTest memWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            memStore.put(key, new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing memory write test.");
    memWriteTest.run(numRequests, numThreads);
    memWriteTest.printStats();
    System.out.println();
    PerformanceTest memReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                memStore.get(String.valueOf(i).getBytes(), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing memory read test.");
    memReadTest.run(numRequests, numThreads);
    memReadTest.printStats();
    System.out.println();
    System.out.println();
    /** * Do Socket tests ** */
    String storeName = "test";
    StoreRepository repository = new StoreRepository();
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    SocketStoreFactory storeFactory = new ClientRequestExecutorPool(10, 1000, 1000, 32 * 1024);
    final Store<ByteArray, byte[], byte[]> socketStore = storeFactory.create(storeName, "localhost", 6666, RequestFormatType.VOLDEMORT_V1, RequestRoutingType.NORMAL);
    RequestHandlerFactory factory = ServerTestUtils.getSocketRequestHandlerFactory(repository);
    AbstractSocketService socketService = ServerTestUtils.getSocketService(useNio, factory, 6666, 50, 50, 1000);
    socketService.start();
    PerformanceTest socketWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] bytes = String.valueOf(i).getBytes();
            ByteArray key = new ByteArray(bytes);
            socketStore.put(key, new Versioned<byte[]>(bytes), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing socket write test.");
    socketWriteTest.run(numRequests, numThreads);
    socketWriteTest.printStats();
    System.out.println();
    PerformanceTest socketReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                socketStore.get(TestUtils.toByteArray(String.valueOf(i)), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing socket read test.");
    socketReadTest.run(numRequests, 1);
    socketReadTest.printStats();
    System.out.println();
    System.out.println();
    socketStore.close();
    storeFactory.close();
    socketService.stop();
    /** * Do HTTP tests ** */
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    HttpService httpService = new HttpService(null, null, repository, RequestFormatType.VOLDEMORT_V0, numThreads, 8080);
    httpService.start();
    ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager(SchemeRegistryFactory.createDefault(), 10000, TimeUnit.MILLISECONDS);
    DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);
    HttpParams clientParams = httpClient.getParams();
    httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
    HttpClientParams.setCookiePolicy(clientParams, CookiePolicy.IGNORE_COOKIES);
    HttpProtocolParams.setUserAgent(clientParams, "test-agent");
    HttpProtocolParams.setVersion(clientParams, HttpVersion.HTTP_1_1);
    HttpConnectionParams.setConnectionTimeout(clientParams, 10000);
    connectionManager.setMaxTotal(numThreads);
    connectionManager.setDefaultMaxPerRoute(numThreads);
    HttpConnectionParams.setStaleCheckingEnabled(clientParams, false);
    final HttpStore httpStore = new HttpStore("test", "localhost", 8080, httpClient, new RequestFormatFactory().getRequestFormat(RequestFormatType.VOLDEMORT_V0), false);
    Thread.sleep(400);
    PerformanceTest httpWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            httpStore.put(new ByteArray(key), new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing HTTP write test.");
    httpWriteTest.run(numRequests, numThreads);
    httpWriteTest.printStats();
    System.out.println();
    PerformanceTest httpReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            httpStore.get(new ByteArray(String.valueOf(i).getBytes()), null);
        }
    };
    System.out.println("Performing HTTP read test.");
    httpReadTest.run(numRequests, numThreads);
    httpReadTest.printStats();
    httpService.stop();
    VoldemortIOUtils.closeQuietly(httpClient);
}

Example 13 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class HintedHandoffFailureTest method customSetup.

/**
     * Setup a cluster with 3 nodes, with the following characteristics:
     * 
     * If FAILURE_MODE is FAIL_FIRST_REPLICA_NODE set the first replica store to
     * a sleepy force failing store
     * 
     * If FAILURE_MODE is FAIL_ALL_REPLICAS: set all replicas to sleepy force
     * failing store
     * 
     * Pseudo master : Standard In-memory store (wrapped by Logging store)
     * 
     * In memory slop stores
     * 
     * @param key The ByteArray representation of the key
     * @param failureMode The Failure mode for the replicas
     * 
     * @throws Exception
     */
public List<Integer> customSetup(ByteArray key, FAILURE_MODE failureMode, ReplicaFactor replicaFactor, long sleepTime, long hintDelayTimeMs) throws Exception {
    cluster = getThreeNodeCluster();
    storeDef = getStoreDef(STORE_NAME, replicaFactor, RoutingStrategyType.CONSISTENT_STRATEGY);
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryStorageEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
    LoggingStore<ByteArray, byte[], byte[]> loggingStore = new LoggingStore<ByteArray, byte[], byte[]>(inMemoryStorageEngine);
    VoldemortException e = new UnreachableStoreException("Node down");
    ForceFailStore<ByteArray, byte[], byte[]> failureStore = new ForceFailStore<ByteArray, byte[], byte[]>(loggingStore, e);
    SleepyStore<ByteArray, byte[], byte[]> sleepyFailureStore = new SleepyStore<ByteArray, byte[], byte[]>(sleepTime, failureStore);
    failureStore.setFail(true);
    List<Integer> failingNodeIdList = Lists.newArrayList();
    List<Node> replicaList = strategy.routeRequest(key.get());
    switch(failureMode) {
        case FAIL_FIRST_REPLICA_NODE:
            failingNodeIdList.add(replicaList.get(1).getId());
            break;
        case FAIL_ALL_REPLICAS:
            for (int nodeId = 1; nodeId < replicaList.size(); nodeId++) {
                failingNodeIdList.add(nodeId);
            }
            break;
    }
    subStores.clear();
    for (int i = 0; i < NUM_NODES_TOTAL; i++) {
        if (failingNodeIdList.contains(i)) {
            subStores.put(i, sleepyFailureStore);
        } else {
            subStores.put(i, loggingStore);
        }
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(NUM_THREADS);
    routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    for (Node node : cluster.getNodes()) {
        int nodeId = node.getId();
        SlopStorageEngine slopStorageEngine = new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(SLOP_STORE_NAME), cluster);
        StorageEngine<ByteArray, Slop, byte[]> storageEngine = slopStorageEngine.asSlopStore();
        nonblockingSlopStores.put(nodeId, routedStoreFactory.toNonblockingStore(slopStorageEngine));
        slopStores.put(nodeId, storageEngine);
    }
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : subStores.entrySet()) nonblockingStores.put(entry.getKey(), routedStoreFactory.toNonblockingStore(entry.getValue()));
    store = new DelayedPutPipelineRoutedStore(subStores, nonblockingStores, slopStores, nonblockingSlopStores, cluster, storeDef, failureDetector, hintDelayTimeMs);
    return failingNodeIdList;
}

Example 14 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class ReadRepairerTest method testMissingKeysAreAddedToNodeWhenDoingReadRepair.

/**
     * See Issue 150: Missing keys are not added to node when performing
     * read-repair
     */
@Test
public void testMissingKeysAreAddedToNodeWhenDoingReadRepair() throws Exception {
    ByteArray key = TestUtils.toByteArray("key");
    byte[] value = "foo".getBytes();
    Cluster cluster = VoldemortTestConstants.getThreeNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 3, 3, 3, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    for (int a = 0; a < 3; a++) {
        int id = Iterables.get(cluster.getNodes(), a).getId();
        InMemoryStorageEngine<ByteArray, byte[], byte[]> subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(id, subStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(failureDetectorClass.getName()).setBannagePeriod(1000).setCluster(cluster).setConnectionVerifier(create(subStores)).setTime(time);
    failureDetector = create(failureDetectorConfig, false);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStore store = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, new RoutedStoreConfig().setTimeoutConfig(new TimeoutConfig(1000L, false)));
    recordException(failureDetector, Iterables.get(cluster.getNodes(), 0));
    store.put(key, new Versioned<byte[]>(value), null);
    recordSuccess(failureDetector, Iterables.get(cluster.getNodes(), 0));
    time.sleep(2000);
    assertEquals(2, store.get(key, null).size());
    // Last get should have repaired the missing key from node 0 so all
    // stores should now return a value
    assertEquals(3, store.get(key, null).size());
    ByteArray anotherKey = TestUtils.toByteArray("anotherKey");
    // Try again, now use getAll to read repair
    recordException(failureDetector, Iterables.get(cluster.getNodes(), 0));
    store.put(anotherKey, new Versioned<byte[]>(value), null);
    recordSuccess(failureDetector, Iterables.get(cluster.getNodes(), 0));
    assertEquals(2, store.getAll(Arrays.asList(anotherKey), null).get(anotherKey).size());
    assertEquals(3, store.get(anotherKey, null).size());
}

Example 15 with InMemoryStorageEngine

use of voldemort.store.memory.InMemoryStorageEngine in project voldemort by voldemort.

the class RoutedStoreTest method testSlowStoreDowngradesFromPreferredToRequired.

@Test
public void testSlowStoreDowngradesFromPreferredToRequired() throws Exception {
    cluster = VoldemortTestConstants.getThreeNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 3, 3, 2, 3, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    int sleepTimeMs = 500;
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        if (subStores.isEmpty()) {
            store = new SleepyStore<ByteArray, byte[], byte[]>(sleepTimeMs, store);
        }
        subStores.put(node.getId(), store);
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(cluster.getNumberOfNodes());
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(10000L));
    routedStore.put(aKey, Versioned.value(aValue), null);
    routedStoreFactory = createFactory();
    routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(sleepTimeMs / 2));
    List<Versioned<byte[]>> versioneds = routedStore.get(aKey, null);
    assertEquals(2, versioneds.size());
}