use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.
the class ReadOnlyStorePerformanceTest method main.
public static void main(String[] args) throws FileNotFoundException, IOException {
OptionParser parser = new OptionParser();
parser.accepts("help", "print usage information");
parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
parser.accepts("requests", "[REQUIRED] number of requests").withRequiredArg().ofType(Integer.class);
parser.accepts("store-dir", "[REQUIRED] store directory").withRequiredArg().describedAs("directory");
parser.accepts("cluster-xml", "Path to cluster.xml").withRequiredArg().describedAs("path");
parser.accepts("node-id", "Id of node").withRequiredArg().ofType(Integer.class).describedAs("node-id");
parser.accepts("search-strategy", "class of the search strategy to use").withRequiredArg().describedAs("class_name");
parser.accepts("build", "If present, first build the data");
parser.accepts("num-values", "The number of values in the store").withRequiredArg().describedAs("count").ofType(Integer.class);
parser.accepts("num-chunks", "The number of chunks per partition").withRequiredArg().describedAs("chunks").ofType(Integer.class);
parser.accepts("internal-sort-size", "The number of items to sort in memory at a time").withRequiredArg().describedAs("size").ofType(Integer.class);
parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
parser.accepts("working-dir", "The directory in which to store temporary data").withRequiredArg().describedAs("dir");
parser.accepts("gzip", "Compress the intermediate temp files used in building the store");
parser.accepts("request-file", "file get request ids from").withRequiredArg();
parser.accepts("version", "Version of read-only store [" + ReadOnlyStorageFormat.READONLY_V0 + "," + ReadOnlyStorageFormat.READONLY_V1 + "," + ReadOnlyStorageFormat.READONLY_V2 + " (default)]").withRequiredArg().describedAs("version");
parser.accepts("test-gz", "Path to gzip containing data. Works with --build only").withRequiredArg().describedAs("path");
OptionSet options = parser.parse(args);
if (options.has("help")) {
parser.printHelpOn(System.out);
System.exit(0);
}
CmdUtils.croakIfMissing(parser, options, "requests", "store-dir");
final int numThreads = CmdUtils.valueOf(options, "threads", 10);
final int numRequests = (Integer) options.valueOf("requests");
final int internalSortSize = CmdUtils.valueOf(options, "internal-sort-size", 500000);
int numValues = numRequests;
final String inputFile = (String) options.valueOf("request-file");
final String searcherClass = CmdUtils.valueOf(options, "search-strategy", BinarySearchStrategy.class.getName()).trim();
final boolean gzipIntermediate = options.has("gzip");
final SearchStrategy searcher = (SearchStrategy) ReflectUtils.callConstructor(ReflectUtils.loadClass(searcherClass));
final File workingDir = new File(CmdUtils.valueOf(options, "working-dir", System.getProperty("java.io.tmpdir")));
String storeDir = (String) options.valueOf("store-dir");
ReadOnlyStorageFormat format = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "version", ReadOnlyStorageFormat.READONLY_V2.toString()));
Cluster cluster = null;
int nodeId = 0;
SerializerDefinition sdef = new SerializerDefinition("json", "'string'");
StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setKeySerializer(sdef).setValueSerializer(sdef).setRequiredReads(1).setReplicationFactor(1).setRequiredWrites(1).setType("read-only").setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setRoutingPolicy(RoutingTier.CLIENT).build();
if (options.has("build")) {
CmdUtils.croakIfMissing(parser, options, "num-values", "value-size");
numValues = (Integer) options.valueOf("num-values");
int numChunks = 1;
if (options.has("num-chunks"))
numChunks = (Integer) options.valueOf("num-chunks");
int valueSize = (Integer) options.valueOf("value-size");
// generate test data
File temp = null;
if (options.has("test-gz")) {
temp = new File((String) options.valueOf("test-gz"));
} else {
temp = File.createTempFile("json-data", ".txt.gz", workingDir);
temp.deleteOnExit();
System.out.println("Generating test data in " + temp);
OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(temp));
Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream), 10 * 1024 * 1024);
String value = TestUtils.randomLetters(valueSize);
for (int i = 0; i < numValues; i++) {
writer.write("\"");
writer.write(Integer.toString(i));
writer.write("\" \"");
writer.write(value);
writer.write("\"");
writer.write("\n");
}
writer.close();
writer = null;
}
System.out.println("Building store.");
InputStream inputStream = new GZIPInputStream(new FileInputStream(temp));
Reader r = new BufferedReader(new InputStreamReader(inputStream), 1 * 1024 * 1024);
File output = TestUtils.createTempDir(workingDir);
File tempDir = TestUtils.createTempDir(workingDir);
cluster = ServerTestUtils.getLocalCluster(1);
nodeId = 0;
JsonStoreBuilder builder = new JsonStoreBuilder(new JsonReader(r), cluster, storeDef, new ConsistentRoutingStrategy(cluster, 1), output, tempDir, internalSortSize, 2, numChunks, 64 * 1024, gzipIntermediate);
builder.build(format);
// copy to store dir
File dir = new File(storeDir);
Utils.rm(dir);
dir.mkdirs();
System.out.println("Moving store data from " + output + " to " + dir);
boolean copyWorked = new File(output, "node-0").renameTo(new File(dir, "version-0"));
if (!copyWorked)
Utils.croak("Copy of data from " + output + " to " + dir + " failed.");
} else {
CmdUtils.croakIfMissing(parser, options, "cluster-xml", "node-id");
String clusterXmlPath = (String) options.valueOf("cluster-xml");
nodeId = (Integer) options.valueOf("node-id");
File clusterXml = new File(clusterXmlPath);
if (!clusterXml.exists()) {
Utils.croak("Cluster.xml does not exist");
}
cluster = new ClusterMapper().readCluster(clusterXml);
}
final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test", searcher, new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster), nodeId, new File(storeDir), 0);
final AtomicInteger obsoletes = new AtomicInteger(0);
final AtomicInteger nullResults = new AtomicInteger(0);
final AtomicInteger totalResults = new AtomicInteger(0);
final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
final Executor executor = Executors.newFixedThreadPool(1);
// if they have given us a file make a request generator that reads from
// it, otherwise just generate random values
final int numVals = numValues;
Runnable requestGenerator;
if (inputFile == null) {
requestGenerator = new Runnable() {
public void run() {
System.out.println("Generating random requests.");
Random random = new Random();
try {
while (true) requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
} catch (InterruptedException e) {
e.printStackTrace();
}
}
};
} else {
requestGenerator = new Runnable() {
public void run() {
try {
System.out.println("Using request file to generate requests.");
BufferedReader reader = new BufferedReader(new FileReader(inputFile), 1000000);
while (true) {
String line = reader.readLine();
if (line == null)
return;
requestIds.put(line.trim());
}
} catch (Exception e) {
e.printStackTrace();
}
}
};
}
executor.execute(requestGenerator);
final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"), true);
final AtomicInteger current = new AtomicInteger();
final int progressIncrement = numRequests / 5;
PerformanceTest readWriteTest = new PerformanceTest() {
@Override
public void doOperation(int index) throws Exception {
try {
totalResults.incrementAndGet();
int curr = current.getAndIncrement();
List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())), null);
if (curr % progressIncrement == 0)
System.out.println(curr);
if (results.size() == 0)
nullResults.incrementAndGet();
} catch (ObsoleteVersionException e) {
obsoletes.incrementAndGet();
}
}
};
System.out.println("Running test...");
readWriteTest.run(numRequests, numThreads);
System.out.println("Random Access Read Only store Results:");
System.out.println("Null reads ratio:" + (nullResults.doubleValue()) / totalResults.doubleValue());
readWriteTest.printStats();
System.exit(0);
}
use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.
the class MultithreadedStressTest method testGetAndPut.
public void testGetAndPut() throws Exception {
final AtomicInteger obsoletes = new AtomicInteger(0);
final CountDownLatch isDone = new CountDownLatch(numberOfRequests);
for (int i = 0; i < numberOfRequests; i++) {
final int index = i % numberOfValues;
service.execute(new Runnable() {
public void run() {
boolean done = false;
while (!done) {
try {
byte[] key = Integer.toString(index).getBytes();
List<Versioned<byte[]>> found = store.get(key, null);
if (found.size() > 1) {
throw new RuntimeException("Found multiple versions: " + found);
} else if (found.size() == 1) {
Versioned<byte[]> versioned = found.get(0);
byte[] valueBytes = Integer.toString(MultithreadedStressTest.this.value.getAndIncrement()).getBytes();
versioned.setObject(valueBytes);
store.put(key, versioned, null);
done = true;
} else if (found.size() == 0) {
throw new RuntimeException("No values found!");
}
} catch (ObsoleteVersionException e) {
obsoletes.getAndIncrement();
} finally {
isDone.countDown();
}
}
}
});
}
isDone.await();
System.err.println("Number of obsoletes: " + obsoletes.get());
System.exit(0);
}
use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.
the class AddNodeAction method update.
/**
* @throws ObsoleteVersionException if a concurrent modification (remove or
* another add) has not completed modifying the structure.
* @throws ArrayIndexOutOfBoundsException if no more ids left to identify
* object
*/
@Override
public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
_storeClient = storeClient;
VListKey<K> newKey = new VListKey<K>(_key, 0);
Versioned<Map<String, Object>> firstNodeMap = storeClient.get(newKey.mapValue());
// adding first node of the list
if (firstNodeMap == null) {
Versioned<Map<String, Object>> newNode = new Versioned<Map<String, Object>>((new VListNode<E>(_value, 0, VStack.NULL_ID, VStack.NULL_ID, true)).mapValue());
// throws ObsoleteVersionException if another process has created a
// new node already
storeClient.put(newKey.mapValue(), newNode);
} else // add to front of list
{
Versioned<VListNode<E>> firstNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(firstNodeMap.getValue()), firstNodeMap.getVersion());
if (!firstNode.getValue().isStable()) {
throw new ObsoleteVersionException("cannot add when list node is not stable");
}
// set stable flag to false
Map<String, Object> tmpMap = new HashMap<String, Object>(firstNodeMap.getValue());
tmpMap.put(VListNode.STABLE, false);
storeClient.put(newKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, firstNodeMap.getVersion()));
_rollback.put(0, firstNodeMap.getValue());
int newId;
int nextId = firstNode.getValue().getNextId();
newId = (nextId == VStack.NULL_ID) ? 1 : nextId + 1;
if (newId == Integer.MAX_VALUE)
throw new ArrayIndexOutOfBoundsException(newId + " out of bounds");
Versioned<VListNode<E>> nextNode = null;
VListKey<K> nextKey = new VListKey<K>(_key, nextId);
if (nextId != VStack.NULL_ID) {
Versioned<Map<String, Object>> nextNodeMap = storeClient.get(nextKey.mapValue());
if (nextNodeMap == null)
throw new ObsoleteVersionException("possible concurrent modification");
nextNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(nextNodeMap.getValue()), nextNodeMap.getVersion());
if (!nextNode.getValue().isStable()) {
throw new ObsoleteVersionException("cannot add when list node is not stable");
}
// set stable flag to false
tmpMap = new HashMap<String, Object>(nextNode.getValue().mapValue());
tmpMap.put(VListNode.STABLE, false);
storeClient.put(nextKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, nextNode.getVersion()));
_rollback.put(nextId, nextNode.getValue().mapValue());
}
// insert new node
Map<String, Object> newNode = (new VListNode<E>(_value, 0, VStack.NULL_ID, newId, true)).mapValue();
// don't need to specify versioned because node is already "locked"
storeClient.put(newKey.mapValue(), newNode);
// move first node to next index
VListKey<K> firstKey = new VListKey<K>(_key, newId);
firstNode = new Versioned<VListNode<E>>(new VListNode<E>(firstNode.getValue().getValue(), newId, 0, firstNode.getValue().getNextId(), true));
// don't need to specify versioned because node is already "locked"
storeClient.put(firstKey.mapValue(), firstNode.getValue().mapValue());
// redefine previous pointer on next node
if (nextNode != null) {
if (!storeClient.applyUpdate(new UpdateNextNode<K, E>(nextNode, nextKey, newId)))
throw new ObsoleteVersionException("unable to update node");
}
}
}
use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.
the class UpdateElementById method update.
@Override
public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
Versioned<Map<String, Object>> nodeMap = storeClient.get(_key.mapValue());
if (nodeMap == null)
throw new IndexOutOfBoundsException("invalid id " + _key.getId());
Version version = (_version != null) ? _version : nodeMap.getVersion();
VListNode<E> listNode = VListNode.valueOf(nodeMap.getValue());
if (!listNode.isStable()) {
throw new ObsoleteVersionException("node " + _key.getId() + " not stable.");
}
_result = listNode.getValue();
VListNode<E> newNode = new VListNode<E>(_element, listNode.getId(), listNode.getPreviousId(), listNode.getNextId(), true);
storeClient.put(_key.mapValue(), new Versioned<Map<String, Object>>(newNode.mapValue(), version));
}
use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.
the class VStack method setById.
/**
* Put the given value to the appropriate id in the stack, using the version
* of the current list node identified by that id.
*
* @param id
* @param element element to set
* @return element that was replaced by the new element
* @throws ObsoleteVersionException when an update fails
*/
public E setById(final int id, final E element) {
VListKey<K> key = new VListKey<K>(_key, id);
UpdateElementById<K, E> updateElementAction = new UpdateElementById<K, E>(key, element);
if (!_storeClient.applyUpdate(updateElementAction))
throw new ObsoleteVersionException("update failed");
return updateElementAction.getResult();
}
Aggregations