Examples with ClientConfig voldemort.client.ClientConfig used on opensource projects

Example 1 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}

Example 2 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class Benchmark method initializeStore.

@SuppressWarnings("unchecked")
public void initializeStore(Props benchmarkProps) throws Exception {
    this.numThreads = benchmarkProps.getInt(THREADS, MAX_WORKERS);
    this.numConnectionsPerNode = benchmarkProps.getInt(NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE);
    this.numIterations = benchmarkProps.getInt(ITERATIONS, 1);
    this.statusIntervalSec = benchmarkProps.getInt(INTERVAL, 0);
    this.verbose = benchmarkProps.getBoolean(VERBOSE, false);
    this.verifyRead = benchmarkProps.getBoolean(VERIFY, false);
    this.ignoreNulls = benchmarkProps.getBoolean(IGNORE_NULLS, false);
    int clientZoneId = benchmarkProps.getInt(CLIENT_ZONE_ID, -1);
    if (benchmarkProps.containsKey(URL)) {
        // Remote benchmark
        if (!benchmarkProps.containsKey(STORE_NAME)) {
            throw new VoldemortException("Missing storename");
        }
        String socketUrl = benchmarkProps.getString(URL);
        String storeName = benchmarkProps.getString(STORE_NAME);
        ClientConfig clientConfig = new ClientConfig().setMaxThreads(numThreads).setMaxTotalConnections(numThreads).setMaxConnectionsPerNode(numConnectionsPerNode).setRoutingTimeout(1500, TimeUnit.MILLISECONDS).setSocketTimeout(1500, TimeUnit.MILLISECONDS).setConnectionTimeout(500, TimeUnit.MILLISECONDS).setRequestFormatType(RequestFormatType.VOLDEMORT_V3).setBootstrapUrls(socketUrl);
        if (clientZoneId >= 0) {
            clientConfig.setClientZoneId(clientZoneId);
        }
        SocketStoreClientFactory socketFactory = new SocketStoreClientFactory(clientConfig);
        this.storeClient = socketFactory.getStoreClient(storeName);
        StoreDefinition storeDef = getStoreDefinition(socketFactory, storeName);
        this.keyType = findKeyType(storeDef);
        benchmarkProps.put(Benchmark.KEY_TYPE, this.keyType);
        this.factory = socketFactory;
    } else {
        // Local benchmark
        localMode = true;
        String storageEngineClass = benchmarkProps.getString(STORAGE_CONFIGURATION_CLASS);
        this.keyType = benchmarkProps.getString(KEY_TYPE, STRING_KEY_TYPE);
        Serializer serializer = findKeyType(this.keyType);
        Store<Object, Object, Object> store = null;
        VoldemortConfig voldemortConfig;
        if (benchmarkProps.containsKey(LOCAL_SERVER_PROPERTIES)) {
            File homeDir = TestUtils.createTempDir();
            File configDir = new File(homeDir, "config");
            configDir.mkdir();
            FileUtils.copyFile(new File(benchmarkProps.get(LOCAL_SERVER_PROPERTIES)), new File(configDir, "server.properties"));
            voldemortConfig = VoldemortConfig.loadFromVoldemortHome(homeDir.getAbsolutePath());
        } else {
            voldemortConfig = ServerTestUtils.getVoldemortConfig();
        }
        StorageConfiguration conf = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { voldemortConfig });
        StorageEngine<ByteArray, byte[], byte[]> engine = conf.getStore(TestUtils.makeStoreDefinition(DUMMY_DB), TestUtils.makeSingleNodeRoutingStrategy());
        if (conf.getType().compareTo(ViewStorageConfiguration.TYPE_NAME) == 0) {
            engine = new ViewStorageEngine(STORE_NAME, engine, new StringSerializer(), new StringSerializer(), serializer, new StringSerializer(), null, BenchmarkViews.loadTransformation(benchmarkProps.getString(VIEW_CLASS).trim()));
        }
        store = SerializingStore.wrap(engine, serializer, new StringSerializer(), new IdentitySerializer());
        this.factory = new StaticStoreClientFactory(store);
        this.storeClient = factory.getStoreClient(store.getName());
    }
    this.storeInitialized = true;
}

Example 3 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class MetadataStressTest method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.err.println("java voldemort.performance.MetadataStressTest url iterations threads selectors");
        System.exit(-1);
    }
    String url = args[0];
    final int count = Integer.parseInt(args[1]);
    int numThreads = Integer.parseInt(args[2]);
    int numSelectors = args.length > 3 ? Integer.parseInt(args[3]) : 8;
    int timeoutSecs = args.length > 4 ? Integer.parseInt(args[4]) : 10;
    ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {

        public Thread newThread(Runnable r) {
            Thread thread = new Thread(r);
            thread.setName("stress-test");
            return thread;
        }
    });
    try {
        final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(url).setEnableLazy(false).setConnectionTimeout(timeoutSecs, TimeUnit.SECONDS).setSocketTimeout(timeoutSecs, TimeUnit.SECONDS).setMaxThreads(numThreads).setSelectors(numSelectors));
        for (int i = 0; i < numThreads; i++) {
            executor.submit(new Runnable() {

                public void run() {
                    for (int j = 0; j < count; j++) {
                        try {
                            String clusterXml = factory.bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY);
                            new ClusterMapper().readCluster(new StringReader(clusterXml));
                            String storesXml = factory.bootstrapMetadataWithRetries(MetadataStore.STORES_KEY);
                            new StoreDefinitionsMapper().readStoreList(new StringReader(storesXml));
                            if (logger.isTraceEnabled())
                                logger.trace("ok " + j);
                        } catch (MappingException me) {
                            logger.fatal(me, me);
                            System.exit(-1);
                        } catch (Exception e) {
                            logger.error(e, e);
                        }
                    }
                }
            });
        }
    } finally {
        executor.shutdown();
    }
}

Example 4 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class SlopStreamingTestEnvironment method run.

@Override
public void run() {
    Random random = new Random(System.currentTimeMillis());
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    bootstrapUrl = cluster.getNodeById(0).getSocketUrl().toString();
    storeDef = storeDefBuilder.build();
    // setup store engines
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        // do only once
        createInnerStore(nodeId);
    }
    for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
        try {
            startServer(nodeId);
        } catch (IOException e) {
            logger.error("Server " + nodeId + "failed to start", e);
        }
    }
    factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl));
    // wait for start of servers
    startFinishLatch.countDown();
    try {
        Set<Integer> nodesToFail = new HashSet<Integer>();
        nodesToFail.add(1);
        boolean wrapUpSignal = false;
        if (logger.isInfoEnabled()) {
            if (wrapUpSignal) {
                logger.info("Wake Up and wrap up. Make all servers NORMAL");
            } else {
                logger.info("Wake Up and decide new failure statuses");
            }
            for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : realStores.entrySet()) {
                InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]> engine = (InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) entry.getValue();
                logger.info("Outstanding Put Assertions of node [" + entry.getKey() + "]: " + engine.getFailedAssertions().size());
            }
        }
        if (logger.isInfoEnabled()) {
            logger.info("Setting nodes to Fail: " + nodesToFail.toString());
        }
        for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
            makeNodeNormal(nodeId);
        }
        for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
            if (nodesToFail.contains(nodeId)) {
                // fail a node if it's normal
                if (nodesStatus.get(nodeId) == NodeStatus.NORMAL) {
                    makeNodeDown(nodeId);
                }
            } else {
                // make node normal if not normal
                if (nodesStatus.get(nodeId) != NodeStatus.NORMAL) {
                    makeNodeNormal(nodeId);
                }
            }
        }
        if (throwIntermittentException)
            waitForFaultyNodeLatch.countDown();
        if (!throwIntermittentException) {
            for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
                makeNodeNormal(nodeId);
            }
        }
    } catch (Exception e) {
    } finally {
        wrapUpFinishLatch.countDown();
    }
}

Example 5 with ClientConfig

use of voldemort.client.ClientConfig in project voldemort by voldemort.

the class E2EClientRequestExecutorPoolAndFailureDetectorTest method setUp.

public void setUp(int opSlowMs, int numSelectors, int connectionsPerNode) throws Exception {
    socketStoreFactory = new ClientRequestExecutorPool(numSelectors, connectionsPerNode, CONNECTION_TIMEOUT_MS, SOCKET_TIMEOUT_MS, SOCKET_BUFFER_SIZE, SOCKET_KEEP_ALIVE);
    cluster = ServerTestUtils.getLocalCluster(numServers, new int[][] { { 0, 4 }, { 1, 5 }, { 2, 6 }, { 3, 7 } });
    servers = new ArrayList<VoldemortServer>();
    Properties p = new Properties();
    String storageConfigs = BdbStorageConfiguration.class.getName() + "," + InMemoryStorageConfiguration.class.getName() + "," + SlowStorageConfiguration.class.getName();
    p.setProperty("storage.configs", storageConfigs);
    p.setProperty("testing.slow.queueing.put.ms", Long.toString(opSlowMs));
    p.setProperty("testing.slow.queueing.get.ms", Long.toString(opSlowMs));
    // Susceptible to BindException...
    for (int i = 0; i < numServers; i++) {
        VoldemortConfig voldemortConfig = ServerTestUtils.createServerConfigWithDefs(this.useNio, i, TestUtils.createTempDir().getAbsolutePath(), cluster, getStoreDef(), p);
        VoldemortServer voldemortServer = ServerTestUtils.startVoldemortServer(socketStoreFactory, voldemortConfig);
        servers.add(voldemortServer);
    }
    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    storeClientFactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl).setMaxConnectionsPerNode(connectionsPerNode).setConnectionTimeout(CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS).setSocketTimeout(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS).setRoutingTimeout(ROUTING_TIMEOUT_MS, TimeUnit.MILLISECONDS).setFailureDetectorThreshold(99).setFailureDetectorThresholdInterval(250));
}