Search in sources :

Example 1 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}
Also used : HashMap(java.util.HashMap) Node(voldemort.cluster.Node) RoutedStoreFactory(voldemort.store.routed.RoutedStoreFactory) RoutedStore(voldemort.store.routed.RoutedStore) Store(voldemort.store.Store) SleepyStore(voldemort.store.SleepyStore) StoreRepository(voldemort.server.StoreRepository) VoldemortServer(voldemort.server.VoldemortServer) Properties(java.util.Properties) OptionParser(joptsimple.OptionParser) VoldemortException(voldemort.VoldemortException) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) StoreDefinition(voldemort.store.StoreDefinition) BannagePeriodFailureDetector(voldemort.cluster.failuredetector.BannagePeriodFailureDetector) FailureDetector(voldemort.cluster.failuredetector.FailureDetector) ByteArray(voldemort.utils.ByteArray) ClientConfig(voldemort.client.ClientConfig) BannagePeriodFailureDetector(voldemort.cluster.failuredetector.BannagePeriodFailureDetector) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) SleepyStore(voldemort.store.SleepyStore) RoutedStore(voldemort.store.routed.RoutedStore) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) RoutedStoreConfig(voldemort.store.routed.RoutedStoreConfig) FailureDetectorConfig(voldemort.cluster.failuredetector.FailureDetectorConfig) ExecutorService(java.util.concurrent.ExecutorService) OptionSet(joptsimple.OptionSet) File(java.io.File)

Example 2 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class AdminServiceFailureTest method getAdminServer.

private AbstractSocketService getAdminServer(Node node, Cluster cluster, List<StoreDefinition> storeDefs, StorageEngine<ByteArray, byte[], byte[]> storageEngine) throws IOException {
    StoreRepository storeRepository = new StoreRepository();
    storeRepository.addStorageEngine(storageEngine);
    storeRepository.addLocalStore(storageEngine);
    SocketRequestHandlerFactory requestHandlerFactory = new SocketRequestHandlerFactory(null, storeRepository, ServerTestUtils.createMetadataStore(cluster, storeDefs), ServerTestUtils.createServerConfig(useNio, 0, TestUtils.createTempDir().getAbsolutePath(), null, null, new Properties()), null, null, null, null);
    return ServerTestUtils.getSocketService(useNio, requestHandlerFactory, node.getAdminPort(), 2, 2, 10000);
}
Also used : SocketRequestHandlerFactory(voldemort.server.protocol.SocketRequestHandlerFactory) StoreRepository(voldemort.server.StoreRepository) Properties(java.util.Properties)

Example 3 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class ServerTestUtils method getJettyServer.

public static Context getJettyServer(String clusterXml, String storesXml, String storeName, RequestFormatType requestFormat, int port) throws Exception {
    StoreRepository repository = getStores(storeName, clusterXml, storesXml);
    // initialize servlet
    Server server = new Server(port);
    server.setSendServerVersion(false);
    Context context = new Context(server, "/", Context.NO_SESSIONS);
    RequestHandler handler = getSocketRequestHandlerFactory(clusterXml, storesXml, repository).getRequestHandler(requestFormat);
    context.addServlet(new ServletHolder(new StoreServlet(handler)), "/stores");
    server.start();
    return context;
}
Also used : Context(org.mortbay.jetty.servlet.Context) Server(org.mortbay.jetty.Server) VoldemortServer(voldemort.server.VoldemortServer) RequestHandler(voldemort.server.protocol.RequestHandler) ServletHolder(org.mortbay.jetty.servlet.ServletHolder) StoreRepository(voldemort.server.StoreRepository) StoreServlet(voldemort.server.http.StoreServlet)

Example 4 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class ServerTestUtils method getStores.

public static StoreRepository getStores(String storeName, String clusterXml, String storesXml) {
    StoreRepository repository = new StoreRepository();
    Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName);
    repository.addLocalStore(store);
    repository.addRoutedStore(store);
    // create new metadata store.
    MetadataStore metadata = createMetadataStore(new ClusterMapper().readCluster(new StringReader(clusterXml)), new StoreDefinitionsMapper().readStoreList(new StringReader(storesXml)));
    repository.addLocalStore(metadata);
    return repository;
}
Also used : MetadataStore(voldemort.store.metadata.MetadataStore) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) StringReader(java.io.StringReader) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) ByteArray(voldemort.utils.ByteArray) StoreRepository(voldemort.server.StoreRepository) ClusterMapper(voldemort.xml.ClusterMapper)

Example 5 with StoreRepository

use of voldemort.server.StoreRepository in project voldemort by voldemort.

the class StorageServiceTest method testStores.

@Test
public void testStores() {
    StoreRepository repo = storage.getStoreRepository();
    for (StoreDefinition def : storeDefs) {
        // test local stores
        assertTrue("Missing local store '" + def.getName() + "'.", repo.hasLocalStore(def.getName()));
        assertEquals(def.getName(), repo.getLocalStore(def.getName()).getName());
        assertTrue("Missing storage engine '" + def.getName() + "'.", repo.hasStorageEngine(def.getName()));
        assertEquals(def.getName(), repo.getStorageEngine(def.getName()).getName());
        Integer[] nodeIds = cluster.getNodeIds().toArray(new Integer[0]);
        for (int index = 0; index < cluster.getNumberOfNodes(); index++) {
            assertTrue("Missing node store '" + def.getName() + "'.", repo.hasNodeStore(def.getName(), nodeIds[index]));
            assertEquals(def.getName(), repo.getNodeStore(def.getName(), nodeIds[index]).getName());
        }
    }
}
Also used : StoreDefinition(voldemort.store.StoreDefinition) StoreRepository(voldemort.server.StoreRepository) Test(org.junit.Test)

Aggregations

StoreRepository (voldemort.server.StoreRepository)14 InMemoryStorageEngine (voldemort.store.memory.InMemoryStorageEngine)6 ByteArray (voldemort.utils.ByteArray)6 ClientRequestExecutorPool (voldemort.store.socket.clientrequest.ClientRequestExecutorPool)5 Before (org.junit.Before)4 VoldemortConfig (voldemort.server.VoldemortConfig)4 VoldemortServer (voldemort.server.VoldemortServer)4 StoreDefinition (voldemort.store.StoreDefinition)4 File (java.io.File)3 Properties (java.util.Properties)3 MockTime (voldemort.MockTime)3 Cluster (voldemort.cluster.Cluster)3 RequestHandlerFactory (voldemort.server.protocol.RequestHandlerFactory)3 SocketStoreFactory (voldemort.store.socket.SocketStoreFactory)3 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)3 StringReader (java.io.StringReader)2 Node (voldemort.cluster.Node)2 SchedulerService (voldemort.common.service.SchedulerService)2 MetadataStore (voldemort.store.metadata.MetadataStore)2 SlopStorageEngine (voldemort.store.slop.SlopStorageEngine)2