use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.
the class RemoteStoreComparisonTest method main.
public static void main(String[] args) throws Exception {
if (args.length != 2)
Utils.croak("USAGE: java " + RemoteStoreComparisonTest.class.getName() + " numRequests numThreads [useNio]");
int numRequests = Integer.parseInt(args[0]);
int numThreads = Integer.parseInt(args[1]);
boolean useNio = args.length > 2 ? args[2].equals("true") : false;
/** * In memory test ** */
final Store<byte[], byte[], byte[]> memStore = new InMemoryStorageEngine<byte[], byte[], byte[]>("test");
PerformanceTest memWriteTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
byte[] key = String.valueOf(i).getBytes();
memStore.put(key, new Versioned<byte[]>(key), null);
}
};
System.out.println("###########################################");
System.out.println("Performing memory write test.");
memWriteTest.run(numRequests, numThreads);
memWriteTest.printStats();
System.out.println();
PerformanceTest memReadTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
try {
memStore.get(String.valueOf(i).getBytes(), null);
} catch (Exception e) {
System.out.println("Failure on i = " + i);
e.printStackTrace();
}
}
};
System.out.println("Performing memory read test.");
memReadTest.run(numRequests, numThreads);
memReadTest.printStats();
System.out.println();
System.out.println();
/** * Do Socket tests ** */
String storeName = "test";
StoreRepository repository = new StoreRepository();
repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
SocketStoreFactory storeFactory = new ClientRequestExecutorPool(10, 1000, 1000, 32 * 1024);
final Store<ByteArray, byte[], byte[]> socketStore = storeFactory.create(storeName, "localhost", 6666, RequestFormatType.VOLDEMORT_V1, RequestRoutingType.NORMAL);
RequestHandlerFactory factory = ServerTestUtils.getSocketRequestHandlerFactory(repository);
AbstractSocketService socketService = ServerTestUtils.getSocketService(useNio, factory, 6666, 50, 50, 1000);
socketService.start();
PerformanceTest socketWriteTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
byte[] bytes = String.valueOf(i).getBytes();
ByteArray key = new ByteArray(bytes);
socketStore.put(key, new Versioned<byte[]>(bytes), null);
}
};
System.out.println("###########################################");
System.out.println("Performing socket write test.");
socketWriteTest.run(numRequests, numThreads);
socketWriteTest.printStats();
System.out.println();
PerformanceTest socketReadTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
try {
socketStore.get(TestUtils.toByteArray(String.valueOf(i)), null);
} catch (Exception e) {
System.out.println("Failure on i = " + i);
e.printStackTrace();
}
}
};
System.out.println("Performing socket read test.");
socketReadTest.run(numRequests, 1);
socketReadTest.printStats();
System.out.println();
System.out.println();
socketStore.close();
storeFactory.close();
socketService.stop();
/** * Do HTTP tests ** */
repository.addLocalStore(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeName));
HttpService httpService = new HttpService(null, null, repository, RequestFormatType.VOLDEMORT_V0, numThreads, 8080);
httpService.start();
ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager(SchemeRegistryFactory.createDefault(), 10000, TimeUnit.MILLISECONDS);
DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);
HttpParams clientParams = httpClient.getParams();
httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
HttpClientParams.setCookiePolicy(clientParams, CookiePolicy.IGNORE_COOKIES);
HttpProtocolParams.setUserAgent(clientParams, "test-agent");
HttpProtocolParams.setVersion(clientParams, HttpVersion.HTTP_1_1);
HttpConnectionParams.setConnectionTimeout(clientParams, 10000);
connectionManager.setMaxTotal(numThreads);
connectionManager.setDefaultMaxPerRoute(numThreads);
HttpConnectionParams.setStaleCheckingEnabled(clientParams, false);
final HttpStore httpStore = new HttpStore("test", "localhost", 8080, httpClient, new RequestFormatFactory().getRequestFormat(RequestFormatType.VOLDEMORT_V0), false);
Thread.sleep(400);
PerformanceTest httpWriteTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
byte[] key = String.valueOf(i).getBytes();
httpStore.put(new ByteArray(key), new Versioned<byte[]>(key), null);
}
};
System.out.println("###########################################");
System.out.println("Performing HTTP write test.");
httpWriteTest.run(numRequests, numThreads);
httpWriteTest.printStats();
System.out.println();
PerformanceTest httpReadTest = new PerformanceTest() {
@Override
public void doOperation(int i) {
httpStore.get(new ByteArray(String.valueOf(i).getBytes()), null);
}
};
System.out.println("Performing HTTP read test.");
httpReadTest.run(numRequests, numThreads);
httpReadTest.printStats();
httpService.stop();
VoldemortIOUtils.closeQuietly(httpClient);
}
use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.
the class HintedHandoffTestEnvironment method startServer.
/**
* Start a server How it works:
*
* 1. create a server using test utilities
*
* 2.Inject prepared test store and storage engine
*
* 3. Inject prepared slop store and storage engine
*
* @param nodeId The node of server to start
* @throws IOException
*/
public void startServer(int nodeId) throws IOException {
if (logger.isInfoEnabled())
logger.info("Starting server of node [" + nodeId + "]");
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
List<StoreDefinition> stores = new ArrayList<StoreDefinition>();
stores.add(storeDef);
// start a voldemort server
VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
config.setNioAdminConnectorSelectors(1);
config.setNioConnectorSelectors(5);
config.setSlopFrequencyMs(DEFAULT_SLOP_PUSH_INTERVAL_S * 1000);
config.setSlopStoreType("memory");
config.setFailureDetectorAsyncRecoveryInterval(DEFAULT_ASYNC_RECOVERY_INTERVAL_S * 1000);
VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config);
socketStoreFactory.close();
voldemortServers.put(nodeId, vs);
VoldemortService vsrv = vs.getService(ServiceType.STORAGE);
StoreRepository sr = ((StorageService) vsrv).getStoreRepository();
// storage engine injection
sr.removeLocalStore(STORE_NAME);
sr.addLocalStore(sleepyStores.get(nodeId));
sr.removeStorageEngine(STORE_NAME);
sr.addStorageEngine((StorageEngine<ByteArray, byte[], byte[]>) realStores.get(nodeId));
// slop stores caching and injection
if (!slopStorageEngines.containsKey(nodeId)) {
SlopStorageEngine slopStorageEngine = sr.getSlopStore();
slopStorageEngines.put(nodeId, slopStorageEngine);
} else {
sr.removeStorageEngine("slop");
sr.removeLocalStore("slop");
sr.addStorageEngine(slopStorageEngines.get(nodeId));
sr.addLocalStore(slopStorageEngines.get(nodeId));
sr.setSlopStore(slopStorageEngines.get(nodeId));
}
}
use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.
the class AbstractZoneAffinityTest method setup.
@Before
public void setup() throws IOException {
byte[] v1_bytes = { (byte) 'V', (byte) '1' };
byte[] v2_bytes = { (byte) 'V', (byte) '2' };
byte[] k1_bytes = { (byte) 'K', (byte) '1' };
byte[] k2_bytes = { (byte) 'K', (byte) '2' };
byte[] k3_bytes = { (byte) 'K', (byte) '3' };
clientConfig = new ClientConfig();
clientConfig.setBootstrapUrls(cluster.getNodes().iterator().next().getSocketUrl().toString());
clientConfig.setClientZoneId(clientZoneId);
setupZoneAffinitySettings();
SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
for (Integer nodeId : cluster.getNodeIds()) {
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties());
VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
vservers.put(nodeId, vs);
socketStoreFactories.put(nodeId, socketStoreFactory);
Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getLocalStore(storeDef.getName());
Node node = cluster.getNodeById(nodeId);
VectorClock version1 = new VectorClock();
version1.incrementVersion(0, System.currentTimeMillis());
VectorClock version2 = version1.incremented(0, System.currentTimeMillis());
if (node.getZoneId() == clientZoneId) {
// local zone
store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
} else {
// remote zone
store.put(new ByteArray(k1_bytes), new Versioned<byte[]>(v2_bytes, version2), null);
store.put(new ByteArray(k2_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
store.put(new ByteArray(k3_bytes), new Versioned<byte[]>(v1_bytes, version1), null);
}
}
client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
}
use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.
the class RebalanceRebootstrapConsistencyTest method setUp.
@Before
public void setUp() throws Exception {
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
int numServers = 2;
rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
/*
* Bug fix: The old code was trying to rename a store during rebalance !
* God knows why Renaming it back to the original store name and
* changing other preferences (required reads = 2)
*/
rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(2).setRequiredReads(2).setPreferredWrites(1).setRequiredWrites(1).build();
List<StoreDefinition> storedefs = new ArrayList<StoreDefinition>();
storedefs.add(rwStoreDefWithReplication);
String storesXmlStr = new StoreDefinitionsMapper().writeStoreList(storedefs);
// create a temp file
File tempStoresXml = ServerTestUtils.createTempFile("tempfile", ".tmp");
BufferedWriter bw = new BufferedWriter(new FileWriter(tempStoresXml));
bw.write(storesXmlStr);
bw.close();
VoldemortServer[] voldemortServers = new VoldemortServer[numServers];
int[][] partitionMap = { { 0, 1 }, {} };
cluster = ServerTestUtils.startVoldemortCluster(numServers, voldemortServers, partitionMap, socketStoreFactory, false, null, tempStoresXml.getAbsolutePath(), new Properties());
servers = Lists.newArrayList();
for (int i = 0; i < numServers; ++i) {
servers.add(voldemortServers[i]);
}
socketUrl = voldemortServers[0].getIdentityNode().getSocketUrl().toString();
bootStrapUrls = new String[1];
bootStrapUrls[0] = socketUrl;
ClientConfig clientConfig = new ClientConfig();
clientConfig.setBootstrapUrls(bootStrapUrls).setClientZoneId(this.CLIENT_ZONE_ID);
SystemStoreClientFactory<String, String> systemStoreFactory = new SystemStoreClientFactory<String, String>(clientConfig);
sysVersionStore = systemStoreFactory.createSystemStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
repository = new SystemStoreRepository(clientConfig);
repository.addSystemStore(sysVersionStore, SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
this.scheduler = new SchedulerService(2, SystemTime.INSTANCE, true);
Callable<Void> rebootstrapCallback = new Callable<Void>() {
@Override
public Void call() throws Exception {
// callbackForClusterChange();
checkConsistentMetadata();
return null;
}
};
// Starting the Version Metadata Manager
this.asyncCheckMetadata = new AsyncMetadataVersionManager(this.repository, rebootstrapCallback, null);
scheduler.schedule(asyncCheckMetadata.getClass().getName(), asyncCheckMetadata, new Date(), 500);
while (maxRetries < 3 && !asyncCheckMetadata.isActive) {
Thread.sleep(500);
maxRetries++;
}
}
use of voldemort.store.socket.clientrequest.ClientRequestExecutorPool in project voldemort by voldemort.
the class ClusterForkLiftToolTest method setUpClusters.
@Before
public void setUpClusters() {
// setup four nodes with one store and one partition
final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
try {
int[][] srcPartitionMap = { { 0 }, { 1 }, { 2 }, { 3 } };
srcServers = new VoldemortServer[4];
srcCluster = ServerTestUtils.startVoldemortCluster(4, srcServers, srcPartitionMap, socketStoreFactory, true, null, SRC_STORES_XML, new Properties());
Node node = srcCluster.getNodeById(0);
srcBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
int[][] dstPartitionMap = { { 0 }, { 1 }, { 2 } };
dstServers = new VoldemortServer[3];
dstCluster = ServerTestUtils.startVoldemortCluster(3, dstServers, dstPartitionMap, socketStoreFactory, true, null, DST_STORES_XML, new Properties());
node = dstCluster.getNodeById(0);
dstBootStrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
kvPairs = ServerTestUtils.createRandomKeyValueString(100);
int keyCount = 0;
for (String key : kvPairs.keySet()) {
if (keyCount == 0)
firstKey = key;
if (keyCount == kvPairs.size() - 1)
lastKey = key;
if (keyCount == kvPairs.size() / 2)
conflictKey = key;
keyCount++;
}
srcAdminClient = new AdminClient(srcCluster);
List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(SRC_STORES_XML));
primaryResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, PRIMARY_RESOLVING_STORE_NAME);
globallyResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, GLOBALLY_RESOLVING_STORE_NAME);
nonResolvingStoreDef = StoreUtils.getStoreDef(storeDefs, MULTIPLE_VERSIONS_STORE_NAME);
srcfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(srcBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
srcPrimaryResolvingStoreClient = srcfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
srcGloballyResolvingStoreClient = srcfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
dstfactory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(dstBootStrapUrl).setSelectors(1).setRoutingTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setSocketTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setConnectionTimeout(1000, java.util.concurrent.TimeUnit.MILLISECONDS).setMaxConnectionsPerNode(1));
dstPrimaryResolvingStoreClient = dstfactory.getStoreClient(PRIMARY_RESOLVING_STORE_NAME);
dstGloballyResolvingStoreClient = dstfactory.getStoreClient(GLOBALLY_RESOLVING_STORE_NAME);
} catch (IOException e) {
e.printStackTrace();
fail("Unexpected exception");
}
}
Aggregations