Examples with ClientRequestExecutorPool voldemort.store.socket.clientrequest.ClientRequestExecutorPool used on opensource projects

Example 16 with ClientRequestExecutorPool

use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.

the class RemoteStoreComparisonTest method main.

public static void main(String[] args) throws Exception {
    if (args.length != 2)
        Utils.croak("USAGE: java " + RemoteStoreComparisonTest.class.getName() + " numRequests numThreads [useNio]");
    int numRequests = Integer.parseInt(args[0]);
    int numThreads = Integer.parseInt(args[1]);
    boolean useNio = args.length > 2 ? args[2].equals("true") : false;
    /** * In memory test ** */
    final Store<byte[], byte[], byte[]> memStore = new InMemoryStorageEngine<byte[], byte[], byte[]>("test");
    PerformanceTest memWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            memStore.put(key, new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing memory write test.");
    memWriteTest.run(numRequests, numThreads);
    memWriteTest.printStats();
    System.out.println();
    PerformanceTest memReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                memStore.get(String.valueOf(i).getBytes(), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing memory read test.");
    memReadTest.run(numRequests, numThreads);
    memReadTest.printStats();
    System.out.println();
    System.out.println();
    /** * Do Socket tests ** */
    String storeName = "test";
    StoreRepository repository = new StoreRepository();
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    SocketStoreFactory storeFactory = new ClientRequestExecutorPool(10, 1000, 1000, 32 * 1024);
    final Store<ByteArray, byte[], byte[]> socketStore = storeFactory.create(storeName, "localhost", 6666, RequestFormatType.VOLDEMORT_V1, RequestRoutingType.NORMAL);
    RequestHandlerFactory factory = ServerTestUtils.getSocketRequestHandlerFactory(repository);
    AbstractSocketService socketService = ServerTestUtils.getSocketService(useNio, factory, 6666, 50, 50, 1000);
    socketService.start();
    PerformanceTest socketWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] bytes = String.valueOf(i).getBytes();
            ByteArray key = new ByteArray(bytes);
            socketStore.put(key, new Versioned<byte[]>(bytes), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing socket write test.");
    socketWriteTest.run(numRequests, numThreads);
    socketWriteTest.printStats();
    System.out.println();
    PerformanceTest socketReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            try {
                socketStore.get(TestUtils.toByteArray(String.valueOf(i)), null);
            } catch (Exception e) {
                System.out.println("Failure on i = " + i);
                e.printStackTrace();
            }
        }
    };
    System.out.println("Performing socket read test.");
    socketReadTest.run(numRequests, 1);
    socketReadTest.printStats();
    System.out.println();
    System.out.println();
    socketStore.close();
    storeFactory.close();
    socketService.stop();
    /** * Do HTTP tests ** */
    repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
    HttpService httpService = new HttpService(null, null, repository, RequestFormatType.VOLDEMORT_V0, numThreads, 8080);
    httpService.start();
    ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager(SchemeRegistryFactory.createDefault(), 10000, TimeUnit.MILLISECONDS);
    DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);
    HttpParams clientParams = httpClient.getParams();
    httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
    HttpClientParams.setCookiePolicy(clientParams, CookiePolicy.IGNORE_COOKIES);
    HttpProtocolParams.setUserAgent(clientParams, "test-agent");
    HttpProtocolParams.setVersion(clientParams, HttpVersion.HTTP_1_1);
    HttpConnectionParams.setConnectionTimeout(clientParams, 10000);
    connectionManager.setMaxTotal(numThreads);
    connectionManager.setDefaultMaxPerRoute(numThreads);
    HttpConnectionParams.setStaleCheckingEnabled(clientParams, false);
    final HttpStore httpStore = new HttpStore("test", "localhost", 8080, httpClient, new RequestFormatFactory().getRequestFormat(RequestFormatType.VOLDEMORT_V0), false);
    Thread.sleep(400);
    PerformanceTest httpWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            byte[] key = String.valueOf(i).getBytes();
            httpStore.put(new ByteArray(key), new Versioned<byte[]>(key), null);
        }
    };
    System.out.println("###########################################");
    System.out.println("Performing HTTP write test.");
    httpWriteTest.run(numRequests, numThreads);
    httpWriteTest.printStats();
    System.out.println();
    PerformanceTest httpReadTest = new PerformanceTest() {

        @Override
        public void doOperation(int i) {
            httpStore.get(new ByteArray(String.valueOf(i).getBytes()), null);
        }
    };
    System.out.println("Performing HTTP read test.");
    httpReadTest.run(numRequests, numThreads);
    httpReadTest.printStats();
    httpService.stop();
    VoldemortIOUtils.closeQuietly(httpClient);
}

Example 17 with ClientRequestExecutorPool

use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.

the class HintedHandoffTestEnvironment method startServer.

/**
     * Start a server How it works:
     * 
     * 1. create a server using test utilities
     * 
     * 2.Inject prepared test store and storage engine
     * 
     * 3. Inject prepared slop store and storage engine
     * 
     * @param nodeId The node of server to start
     * @throws IOException
     */
public void startServer(int nodeId) throws IOException {
    if (logger.isInfoEnabled())
        logger.info("Starting server of node [" + nodeId + "]");
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
    List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
    stores.add(storeDef);
    // start a voldemort server
    VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
    config.setNioAdminConnectorSelectors(1);
    config.setNioConnectorSelectors(5);
    config.setSlopFrequencyMs(DEFAULT_SLOP_PUSH_INTERVAL_S * 1000);
    config.setSlopStoreType("memory");
    config.setFailureDetectorAsyncRecoveryInterval(DEFAULT_ASYNC_RECOVERY_INTERVAL_S * 1000);
    VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
    socketStoreFactory.close();
    voldemortServers.put(nodeId, vs);
    VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
    StoreRepository sr = ((StorageService) vsrv).getStoreRepository();
    // storage engine injection
    sr.removeLocalStore(STORE_NAME);
    sr.addLocalStore(sleepyStores.get(nodeId));
    sr.removeStorageEngine(STORE_NAME);
    sr.addStorageEngine((StorageEngine<ByteArray, byte[], byte[]>) realStores.get(nodeId));
    // slop stores caching and injection
    if (!slopStorageEngines.containsKey(nodeId)) {
        SlopStorageEngine slopStorageEngine = sr.getSlopStore();
        slopStorageEngines.put(nodeId, slopStorageEngine);
    } else {
        sr.removeStorageEngine("slop");
        sr.removeLocalStore("slop");
        sr.addStorageEngine(slopStorageEngines.get(nodeId));
        sr.addLocalStore(slopStorageEngines.get(nodeId));
        sr.setSlopStore(slopStorageEngines.get(nodeId));
    }
}

Example 18 with ClientRequestExecutorPool

use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.

the class AbstractZoneAffinityTest method setup.

@Before
public void setup() throws IOException {
    byte[] v1_bytes = { (byte) 'V', (byte) '1' };
    byte[] v2_bytes = { (byte) 'V', (byte) '2' };
    byte[] k1_bytes = { (byte) 'K', (byte) '1' };
    byte[] k2_bytes = { (byte) 'K', (byte) '2' };
    byte[] k3_bytes = { (byte) 'K', (byte) '3' };
    clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(cluster.getNodes().iterator().next().getSocketUrl().toString());
    clientConfig.setClientZoneId(clientZoneId);
    setupZoneAffinitySettings();
    SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
    for (Integer nodeId : cluster.getNodeIds()) {
        SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
        VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
        VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
        vservers.put(nodeId, vs);
        socketStoreFactories.put(nodeId, socketStoreFactory);
        Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getLocalStore(storeDef.getName());
        Node node = cluster.getNodeById(nodeId);
        VectorClock version1 = new VectorClock();
        version1.incrementVersion(0, System.currentTimeMillis());
        VectorClock version2 = version1.incremented(0, System.currentTimeMillis());
        if (node.getZoneId() == clientZoneId) {
            // local zone
            store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
            store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
        } else {
            // remote zone
            store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v2_bytes, version2), null);
            store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
            store.put(new ByteArray(k3_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
        }
    }
    client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
}

Example 19 with ClientRequestExecutorPool

use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.

the class RebalanceRebootstrapConsistencyTest method setUp.

@Before
public void setUp() throws Exception {
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    int numServers = 2;
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    /*
         * Bug fix: The old code was trying to rename a store during rebalance !
         * God knows why Renaming it back to the original store name and
         * changing other preferences (required reads = 2)
         */
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(2).setRequiredReads(2).setPreferredWrites(1).setRequiredWrites(1).build();
    List<StoreDefinition> storedefs = new ArrayList<StoreDefinition>();
    storedefs.add(rwStoreDefWithReplication);
    String storesXmlStr = new StoreDefinitionsMapper().writeStoreList(storedefs);
    // create a temp file
    File tempStoresXml = ServerTestUtils.createTempFile("tempfile", ".tmp");
    BufferedWriter bw = new BufferedWriter(new FileWriter(tempStoresXml));
    bw.write(storesXmlStr);
    bw.close();
    VoldemortServer[] voldemortServers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1 }, {} };
    cluster = ServerTestUtils.startVoldemortCluster(numServers, voldemortServers, partitionMap, socketStoreFactory, false, null, tempStoresXml.getAbsolutePath(), new Properties());
    servers = Lists.newArrayList();
    for (int i = 0; i < numServers; ++i) {
        servers.add(voldemortServers[i]);
    }
    socketUrl = voldemortServers[0].getIdentityNode().getSocketUrl().toString();
    bootStrapUrls = new String[1];
    bootStrapUrls[0] = socketUrl;
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(bootStrapUrls).setClientZoneId(this.CLIENT_ZONE_ID);
    SystemStoreClientFactory<String, String> systemStoreFactory = new SystemStoreClientFactory<String, String>(clientConfig);
    sysVersionStore = systemStoreFactory.createSystemStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    repository = new SystemStoreRepository(clientConfig);
    repository.addSystemStore(sysVersionStore, SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    this.scheduler = new SchedulerService(2, SystemTime.INSTANCE, true);
    Callable<Void> rebootstrapCallback = new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            // callbackForClusterChange();
            checkConsistentMetadata();
            return null;
        }
    };
    // Starting the Version Metadata Manager
    this.asyncCheckMetadata = new AsyncMetadataVersionManager(this.repository, rebootstrapCallback, null);
    scheduler.schedule(asyncCheckMetadata.getClass().getName(), asyncCheckMetadata, new Date(), 500);
    while (maxRetries < 3 && !asyncCheckMetadata.isActive) {
        Thread.sleep(500);
        maxRetries++;
    }
}

Example 20 with ClientRequestExecutorPool

use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.

the class ClusterForkLiftToolTest method setUpClusters.

@Before
public void setUpClusters() {
    // setup four nodes with one store and one partition
    final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    try {
        int[][] srcPartitionMap = { { 0 }, { 1 }, { 2 }, { 3 } };
        srcServers = new VoldemortServer[4];
        srcCluster = ServerTestUtils.startVoldemortCluster(4, srcServers, srcPartitionMap, socketStoreFactory, true, null, SRC_STORES_XML, new Properties());
        Node node = srcCluster.getNodeById(0);
        srcBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
        int[][] dstPartitionMap = { { 0 }, { 1 }, { 2 } };
        dstServers = new VoldemortServer[3];
        dstCluster = ServerTestUtils.startVoldemortCluster(3, dstServers, dstPartitionMap, socketStoreFactory, true, null, DST_STORES_XML, new Properties());
        node = dstCluster.getNodeById(0);
        dstBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
        kvPairs = ServerTestUtils.createRandomKeyValueString(100);
        int keyCount = 0;
        for (String key : kvPairs.keySet()) {
            if (keyCount == 0)
                firstKey = key;
            if (keyCount == kvPairs.size() - 1)
                lastKey = key;
            if (keyCount == kvPairs.size() / 2)
                conflictKey = key;
            keyCount++;
        }
        srcAdminClient = new AdminClient(srcCluster);
        List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(SRC_STORES_XML));
        primaryResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, PRIMARY_RESOLVING_STORE_NAME);
        globallyResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, GLOBALLY_RESOLVING_STORE_NAME);
        nonResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, MULTIPLE_VERSIONS_STORE_NAME);
        srcfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(srcBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
        srcPrimaryResolvingStoreClient = srcfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
        srcGloballyResolvingStoreClient = srcfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
        dstfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(dstBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
        dstPrimaryResolvingStoreClient = dstfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
        dstGloballyResolvingStoreClient = dstfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
    } catch (IOException e) {
        e.printStackTrace();
        fail("Unexpected exception");
    }
}