Examples with StoreDefinitionsMapper voldemort.xml.StoreDefinitionsMapper used on opensource projects

Example 6 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}

Example 7 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class MetadataStressTest method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.err.println("java voldemort.performance.MetadataStressTest url iterations threads selectors");
        System.exit(-1);
    }
    String url = args[0];
    final int count = Integer.parseInt(args[1]);
    int numThreads = Integer.parseInt(args[2]);
    int numSelectors = args.length > 3 ? Integer.parseInt(args[3]) : 8;
    int timeoutSecs = args.length > 4 ? Integer.parseInt(args[4]) : 10;
    ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {

        public Thread newThread(Runnable r) {
            Thread thread = new Thread(r);
            thread.setName("stress-test");
            return thread;
        }
    });
    try {
        final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(url).setEnableLazy(false).setConnectionTimeout(timeoutSecs, TimeUnit.SECONDS).setSocketTimeout(timeoutSecs, TimeUnit.SECONDS).setMaxThreads(numThreads).setSelectors(numSelectors));
        for (int i = 0; i < numThreads; i++) {
            executor.submit(new Runnable() {

                public void run() {
                    for (int j = 0; j < count; j++) {
                        try {
                            String clusterXml = factory.bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY);
                            new ClusterMapper().readCluster(new StringReader(clusterXml));
                            String storesXml = factory.bootstrapMetadataWithRetries(MetadataStore.STORES_KEY);
                            new StoreDefinitionsMapper().readStoreList(new StringReader(storesXml));
                            if (logger.isTraceEnabled())
                                logger.trace("ok " + j);
                        } catch (MappingException me) {
                            logger.fatal(me, me);
                            System.exit(-1);
                        } catch (Exception e) {
                            logger.error(e, e);
                        }
                    }
                }
            });
        }
    } finally {
        executor.shutdown();
    }
}

Example 8 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AdminServiceBasicTest method testFetchSingleStoreFromMetadataStore.

@Test
public void testFetchSingleStoreFromMetadataStore() throws Exception {
    String storeName = "test-replication-memory";
    String storeDefStr = bootstrapMetadata(storeName);
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    List<StoreDefinition> storeDefList = mapper.readStoreList(new StringReader(storeDefStr));
    assertEquals(storeDefList.size(), 1);
    StoreDefinition storeDef = storeDefList.get(0);
    assertEquals(storeDef.getName(), storeName);
}

Example 9 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class AdminServiceBasicTest method updateAndResetStoreDefinitions.

/**
     * Function to update the given stores and then reset the stores.xml back to
     * its original state. This is used for confirming that the updates only
     * affect the specified stores in the server. Rest of the stores remain
     * untouched.
     * 
     * @param storesToBeUpdatedList specifies list of stores to be updated
     */
private void updateAndResetStoreDefinitions(List<StoreDefinition> storesToBeUpdatedList) {
    // Track the names of the stores to be updated
    Set<String> storesNamesToBeUpdated = getStoreNames(storesToBeUpdatedList);
    // Keep track of the original store definitions for the specific stores
    // about to be updated
    List<StoreDefinition> originalStoreDefinitionsList = new ArrayList<StoreDefinition>();
    for (StoreDefinition def : this.storeDefs) {
        if (storesNamesToBeUpdated.contains(def.getName())) {
            originalStoreDefinitionsList.add(def);
        }
    }
    // Update the definitions on all the nodes
    AdminClient adminClient = getAdminClient();
    adminClient.metadataMgmtOps.updateRemoteStoreDefList(storesToBeUpdatedList);
    // Retrieve stores list and check that other definitions are unchanged
    String allStoresDefStr = bootstrapMetadata(MetadataStore.STORES_KEY);
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    List<StoreDefinition> storeDefList = mapper.readStoreList(new StringReader(allStoresDefStr));
    assertEquals(storeDefList.size(), this.storeDefs.size());
    // Insert original stores in the map
    Map<String, StoreDefinition> storeNameToDefMap = new HashMap<String, StoreDefinition>();
    for (StoreDefinition def : this.storeDefs) {
        storeNameToDefMap.put(def.getName(), def);
    }
    // definition should be different. Everything else should be as is
    for (StoreDefinition def : storeDefList) {
        if (!storesNamesToBeUpdated.contains(def.getName())) {
            assertEquals(def, storeNameToDefMap.get(def.getName()));
        }
    }
    // Reset the store definition back to original
    adminClient.metadataMgmtOps.updateRemoteStoreDefList(originalStoreDefinitionsList, this.cluster.getNodeIds());
}

Example 10 with StoreDefinitionsMapper

use of voldemort.xml.StoreDefinitionsMapper in project voldemort by voldemort.

the class Benchmark method getStoreDefinition.

private StoreDefinition getStoreDefinition(AbstractStoreClientFactory factory, String storeName) {
    String storesXml = factory.bootstrapMetadataWithRetries(MetadataStore.STORES_KEY);
    StoreDefinitionsMapper storeMapper = new StoreDefinitionsMapper();
    List<StoreDefinition> storeDefinitionList = storeMapper.readStoreList(new StringReader(storesXml));
    StoreDefinition storeDef = null;
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        if (storeName.equals(storeDefinition.getName())) {
            storeDef = storeDefinition;
        }
    }
    return storeDef;
}