use of voldemort.cluster.Cluster in project voldemort by voldemort.
the class RequestFileFilter method main.
/**
* Filter requests specified in a file, generating a new file containing
* only requests destined for a specific node.
*
* @param args See usage for more information
* @throws Exception In case of I/O or Voldemort-specific errors
*/
public static void main(String[] args) throws Exception {
OptionParser parser = new OptionParser();
parser.accepts("help", "print usage information");
parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
parser.accepts("string-keys");
OptionSet options = parser.parse(args);
if (options.has("help")) {
parser.printHelpOn(System.out);
System.exit(0);
}
Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
if (missing.size() > 0) {
System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
parser.printHelpOn(System.err);
System.exit(1);
}
int nodeId = (Integer) options.valueOf("node");
String storeName = (String) options.valueOf("store-name");
String bootstrapURL = (String) options.valueOf("url");
String inputFile = (String) options.valueOf("input");
String outputFile = (String) options.valueOf("output");
boolean stringKeys = options.has("string-keys");
AdminClient adminClient = new AdminClient(bootstrapURL);
List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
StoreDefinition storeDefinition = null;
for (StoreDefinition def : storeDefinitionList) {
if (storeName.equals(def.getName())) {
storeDefinition = def;
}
}
if (storeDefinition == null) {
Utils.croak("No store found with name\"" + storeName + "\"");
}
Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
Node node = null;
try {
node = cluster.getNodeById(nodeId);
} catch (VoldemortException e) {
Utils.croak("Can't find a node with id " + nodeId);
}
RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
try {
new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
} catch (FileNotFoundException e) {
Utils.croak(e.getMessage());
}
}
use of voldemort.cluster.Cluster in project voldemort by voldemort.
the class RoutedStoreParallelismTest method main.
public static void main(String[] args) throws Throwable {
OptionParser parser = new OptionParser();
parser.accepts("num-keys", "The number of keys to submit for retrieval Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\" Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\" Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
parser.accepts("num-nodes", "The number of nodes Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
parser.accepts("num-clients", "The number of threads to make requests concurrently Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\" Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
parser.accepts("help", "This help");
OptionSet options = parser.parse(args);
if (options.has("help")) {
printUsage(System.out, parser);
}
final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
System.err.println("num-keys : " + numKeys);
System.err.println("max-connections : " + maxConnectionsPerNode);
System.err.println("max-threads : " + maxThreads);
System.err.println("num-nodes : " + numNodes);
System.err.println("num-slow-nodes : " + numSlowNodes);
System.err.println("delay : " + delay);
System.err.println("num-clients : " + numClients);
ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
VoldemortServer[] servers = new VoldemortServer[numNodes];
int[][] partitionMap = new int[numNodes][1];
for (int i = 0; i < numNodes; i++) {
partitionMap[i][0] = i;
}
Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
serverMap.put(i, servers[i]);
Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
if (i < numSlowNodes)
store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
StoreRepository storeRepository = servers[i].getStoreRepository();
storeRepository.addLocalStore(store);
}
Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
for (Node node : cluster.getNodes()) {
Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
stores.put(node.getId(), socketStore);
}
FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
ExecutorService runner = Executors.newFixedThreadPool(numClients);
long start = System.nanoTime();
try {
for (int i = 0; i < numClients; i++) {
runner.submit(new Runnable() {
public void run() {
for (int i = 0; i < numKeys; i++) {
ByteArray key = new ByteArray(("test-key-" + i).getBytes());
try {
routedStore.get(key, null);
} catch (VoldemortException e) {
//
}
}
}
});
}
runner.shutdown();
runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
long time = (System.nanoTime() - start) / Time.NS_PER_MS;
System.err.println("Time: " + time + " ms.");
} finally {
runner.shutdown();
}
if (failureDetector != null)
failureDetector.destroy();
for (VoldemortServer server : serverMap.values()) server.stop();
if (routedStoreThreadPool != null)
routedStoreThreadPool.shutdown();
System.exit(0);
}
use of voldemort.cluster.Cluster in project voldemort by voldemort.
the class ReadOnlyStorePerformanceTest method main.
public static void main(String[] args) throws FileNotFoundException, IOException {
OptionParser parser = new OptionParser();
parser.accepts("help", "print usage information");
parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
parser.accepts("requests", "[REQUIRED] number of requests").withRequiredArg().ofType(Integer.class);
parser.accepts("store-dir", "[REQUIRED] store directory").withRequiredArg().describedAs("directory");
parser.accepts("cluster-xml", "Path to cluster.xml").withRequiredArg().describedAs("path");
parser.accepts("node-id", "Id of node").withRequiredArg().ofType(Integer.class).describedAs("node-id");
parser.accepts("search-strategy", "class of the search strategy to use").withRequiredArg().describedAs("class_name");
parser.accepts("build", "If present, first build the data");
parser.accepts("num-values", "The number of values in the store").withRequiredArg().describedAs("count").ofType(Integer.class);
parser.accepts("num-chunks", "The number of chunks per partition").withRequiredArg().describedAs("chunks").ofType(Integer.class);
parser.accepts("internal-sort-size", "The number of items to sort in memory at a time").withRequiredArg().describedAs("size").ofType(Integer.class);
parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
parser.accepts("working-dir", "The directory in which to store temporary data").withRequiredArg().describedAs("dir");
parser.accepts("gzip", "Compress the intermediate temp files used in building the store");
parser.accepts("request-file", "file get request ids from").withRequiredArg();
parser.accepts("version", "Version of read-only store [" + ReadOnlyStorageFormat.READONLY_V0 + "," + ReadOnlyStorageFormat.READONLY_V1 + "," + ReadOnlyStorageFormat.READONLY_V2 + " (default)]").withRequiredArg().describedAs("version");
parser.accepts("test-gz", "Path to gzip containing data. Works with --build only").withRequiredArg().describedAs("path");
OptionSet options = parser.parse(args);
if (options.has("help")) {
parser.printHelpOn(System.out);
System.exit(0);
}
CmdUtils.croakIfMissing(parser, options, "requests", "store-dir");
final int numThreads = CmdUtils.valueOf(options, "threads", 10);
final int numRequests = (Integer) options.valueOf("requests");
final int internalSortSize = CmdUtils.valueOf(options, "internal-sort-size", 500000);
int numValues = numRequests;
final String inputFile = (String) options.valueOf("request-file");
final String searcherClass = CmdUtils.valueOf(options, "search-strategy", BinarySearchStrategy.class.getName()).trim();
final boolean gzipIntermediate = options.has("gzip");
final SearchStrategy searcher = (SearchStrategy) ReflectUtils.callConstructor(ReflectUtils.loadClass(searcherClass));
final File workingDir = new File(CmdUtils.valueOf(options, "working-dir", System.getProperty("java.io.tmpdir")));
String storeDir = (String) options.valueOf("store-dir");
ReadOnlyStorageFormat format = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "version", ReadOnlyStorageFormat.READONLY_V2.toString()));
Cluster cluster = null;
int nodeId = 0;
SerializerDefinition sdef = new SerializerDefinition("json", "'string'");
StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setKeySerializer(sdef).setValueSerializer(sdef).setRequiredReads(1).setReplicationFactor(1).setRequiredWrites(1).setType("read-only").setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setRoutingPolicy(RoutingTier.CLIENT).build();
if (options.has("build")) {
CmdUtils.croakIfMissing(parser, options, "num-values", "value-size");
numValues = (Integer) options.valueOf("num-values");
int numChunks = 1;
if (options.has("num-chunks"))
numChunks = (Integer) options.valueOf("num-chunks");
int valueSize = (Integer) options.valueOf("value-size");
// generate test data
File temp = null;
if (options.has("test-gz")) {
temp = new File((String) options.valueOf("test-gz"));
} else {
temp = File.createTempFile("json-data", ".txt.gz", workingDir);
temp.deleteOnExit();
System.out.println("Generating test data in " + temp);
OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(temp));
Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream), 10 * 1024 * 1024);
String value = TestUtils.randomLetters(valueSize);
for (int i = 0; i < numValues; i++) {
writer.write("\"");
writer.write(Integer.toString(i));
writer.write("\" \"");
writer.write(value);
writer.write("\"");
writer.write("\n");
}
writer.close();
writer = null;
}
System.out.println("Building store.");
InputStream inputStream = new GZIPInputStream(new FileInputStream(temp));
Reader r = new BufferedReader(new InputStreamReader(inputStream), 1 * 1024 * 1024);
File output = TestUtils.createTempDir(workingDir);
File tempDir = TestUtils.createTempDir(workingDir);
cluster = ServerTestUtils.getLocalCluster(1);
nodeId = 0;
JsonStoreBuilder builder = new JsonStoreBuilder(new JsonReader(r), cluster, storeDef, new ConsistentRoutingStrategy(cluster, 1), output, tempDir, internalSortSize, 2, numChunks, 64 * 1024, gzipIntermediate);
builder.build(format);
// copy to store dir
File dir = new File(storeDir);
Utils.rm(dir);
dir.mkdirs();
System.out.println("Moving store data from " + output + " to " + dir);
boolean copyWorked = new File(output, "node-0").renameTo(new File(dir, "version-0"));
if (!copyWorked)
Utils.croak("Copy of data from " + output + " to " + dir + " failed.");
} else {
CmdUtils.croakIfMissing(parser, options, "cluster-xml", "node-id");
String clusterXmlPath = (String) options.valueOf("cluster-xml");
nodeId = (Integer) options.valueOf("node-id");
File clusterXml = new File(clusterXmlPath);
if (!clusterXml.exists()) {
Utils.croak("Cluster.xml does not exist");
}
cluster = new ClusterMapper().readCluster(clusterXml);
}
final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test", searcher, new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster), nodeId, new File(storeDir), 0);
final AtomicInteger obsoletes = new AtomicInteger(0);
final AtomicInteger nullResults = new AtomicInteger(0);
final AtomicInteger totalResults = new AtomicInteger(0);
final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
final Executor executor = Executors.newFixedThreadPool(1);
// if they have given us a file make a request generator that reads from
// it, otherwise just generate random values
final int numVals = numValues;
Runnable requestGenerator;
if (inputFile == null) {
requestGenerator = new Runnable() {
public void run() {
System.out.println("Generating random requests.");
Random random = new Random();
try {
while (true) requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
} catch (InterruptedException e) {
e.printStackTrace();
}
}
};
} else {
requestGenerator = new Runnable() {
public void run() {
try {
System.out.println("Using request file to generate requests.");
BufferedReader reader = new BufferedReader(new FileReader(inputFile), 1000000);
while (true) {
String line = reader.readLine();
if (line == null)
return;
requestIds.put(line.trim());
}
} catch (Exception e) {
e.printStackTrace();
}
}
};
}
executor.execute(requestGenerator);
final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"), true);
final AtomicInteger current = new AtomicInteger();
final int progressIncrement = numRequests / 5;
PerformanceTest readWriteTest = new PerformanceTest() {
@Override
public void doOperation(int index) throws Exception {
try {
totalResults.incrementAndGet();
int curr = current.getAndIncrement();
List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())), null);
if (curr % progressIncrement == 0)
System.out.println(curr);
if (results.size() == 0)
nullResults.incrementAndGet();
} catch (ObsoleteVersionException e) {
obsoletes.incrementAndGet();
}
}
};
System.out.println("Running test...");
readWriteTest.run(numRequests, numThreads);
System.out.println("Random Access Read Only store Results:");
System.out.println("Null reads ratio:" + (nullResults.doubleValue()) / totalResults.doubleValue());
readWriteTest.printStats();
System.exit(0);
}
use of voldemort.cluster.Cluster in project voldemort by voldemort.
the class AdminServiceBasicTest method testUpdateClusterMetadata.
@Test
public void testUpdateClusterMetadata() {
Cluster updatedCluster = ServerTestUtils.getLocalCluster(4);
AdminClient client = getAdminClient();
for (int i = 0; i < NUM_RUNS; i++) {
VectorClock existingClock = ((VectorClock) client.metadataMgmtOps.getRemoteCluster(0).getVersion());
VectorClock clock = existingClock.incremented(0, System.currentTimeMillis());
client.metadataMgmtOps.updateRemoteCluster(0, updatedCluster, clock);
assertEquals("Cluster should match", updatedCluster, getVoldemortServer(0).getMetadataStore().getCluster());
assertEquals("AdminClient.getMetdata() should match", client.metadataMgmtOps.getRemoteCluster(0).getValue(), updatedCluster);
// version should match
assertEquals("versions should match as well.", clock, client.metadataMgmtOps.getRemoteCluster(0).getVersion());
}
}
use of voldemort.cluster.Cluster in project voldemort by voldemort.
the class AdminServiceBasicTest method testReplicationMappingWithZonePreference.
@Test
public void testReplicationMappingWithZonePreference() {
List<Zone> zones = ServerTestUtils.getZones(2);
List<Node> nodes = Lists.newArrayList();
nodes.add(new Node(0, "localhost", 1, 2, 3, 0, Lists.newArrayList(0, 4, 8)));
nodes.add(new Node(1, "localhost", 1, 2, 3, 0, Lists.newArrayList(1, 5, 9)));
nodes.add(new Node(2, "localhost", 1, 2, 3, 1, Lists.newArrayList(2, 6, 10)));
nodes.add(new Node(3, "localhost", 1, 2, 3, 1, Lists.newArrayList(3, 7, 11)));
// Test 0 - With rep-factor 1; zone 1
StoreDefinition storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
Cluster newCluster = new Cluster("single_zone_cluster", nodes, zones);
try {
adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
fail("Should have thrown an exception since rep-factor = 1");
} catch (VoldemortException e) {
}
// With rep-factor 1; zone 0
storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
newCluster = new Cluster("single_zone_cluster", nodes, zones);
try {
adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
fail("Should have thrown an exception since rep-factor = 1");
} catch (VoldemortException e) {
}
// Test 1 - With consistent routing strategy
storeDef = ServerTestUtils.getStoreDef("consistent", 4, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
// On node 0; zone id 1
Map<Integer, List<Integer>> replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
{
HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10));
expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
assertEquals(expectedMapping, replicationMapping);
}
// On node 0; zone id 0
replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
{
HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
// partitionTuple.put(1, Lists.newArrayList(0, 4, 8));
// partitionTuple.put(2, Lists.newArrayList(3, 7, 11));
// partitionTuple.put(3, Lists.newArrayList(2, 6, 10));
expectedMapping.put(1, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11));
assertEquals(expectedMapping, replicationMapping);
}
// Test 2 - With zone routing strategy, and zone replication factor 1
HashMap<Integer, Integer> zoneReplicationFactors = Maps.newHashMap();
for (int zoneIds = 0; zoneIds < 2; zoneIds++) {
zoneReplicationFactors.put(zoneIds, 1);
}
storeDef = ServerTestUtils.getStoreDef("zone", 2, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
newCluster = new Cluster("multi_zone_cluster", nodes, zones);
{
try {
replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
fail("Should have thrown an exception since zoneReplicationFactor is 1");
} catch (VoldemortException e) {
}
}
{
// On node 0, zone 1
replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 2, 6, 10));
expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
assertEquals(expectedMapping, replicationMapping);
}
{
// On node 1, zone 1
replicationMapping = adminClient.replicaOps.getReplicationMapping(1, newCluster, storeDef, 1);
HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
expectedMapping.put(2, Lists.newArrayList(1, 5, 9));
assertEquals(expectedMapping, replicationMapping);
}
}
Aggregations