Search in sources :

Example 6 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class ReadOnlyStorePerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("requests", "[REQUIRED] number of requests").withRequiredArg().ofType(Integer.class);
    parser.accepts("store-dir", "[REQUIRED] store directory").withRequiredArg().describedAs("directory");
    parser.accepts("cluster-xml", "Path to cluster.xml").withRequiredArg().describedAs("path");
    parser.accepts("node-id", "Id of node").withRequiredArg().ofType(Integer.class).describedAs("node-id");
    parser.accepts("search-strategy", "class of the search strategy to use").withRequiredArg().describedAs("class_name");
    parser.accepts("build", "If present, first build the data");
    parser.accepts("num-values", "The number of values in the store").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts("num-chunks", "The number of chunks per partition").withRequiredArg().describedAs("chunks").ofType(Integer.class);
    parser.accepts("internal-sort-size", "The number of items to sort in memory at a time").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("working-dir", "The directory in which to store temporary data").withRequiredArg().describedAs("dir");
    parser.accepts("gzip", "Compress the intermediate temp files used in building the store");
    parser.accepts("request-file", "file get request ids from").withRequiredArg();
    parser.accepts("version", "Version of read-only store [" + ReadOnlyStorageFormat.READONLY_V0 + "," + ReadOnlyStorageFormat.READONLY_V1 + "," + ReadOnlyStorageFormat.READONLY_V2 + " (default)]").withRequiredArg().describedAs("version");
    parser.accepts("test-gz", "Path to gzip containing data. Works with --build only").withRequiredArg().describedAs("path");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    CmdUtils.croakIfMissing(parser, options, "requests", "store-dir");
    final int numThreads = CmdUtils.valueOf(options, "threads", 10);
    final int numRequests = (Integer) options.valueOf("requests");
    final int internalSortSize = CmdUtils.valueOf(options, "internal-sort-size", 500000);
    int numValues = numRequests;
    final String inputFile = (String) options.valueOf("request-file");
    final String searcherClass = CmdUtils.valueOf(options, "search-strategy", BinarySearchStrategy.class.getName()).trim();
    final boolean gzipIntermediate = options.has("gzip");
    final SearchStrategy searcher = (SearchStrategy) ReflectUtils.callConstructor(ReflectUtils.loadClass(searcherClass));
    final File workingDir = new File(CmdUtils.valueOf(options, "working-dir", System.getProperty("java.io.tmpdir")));
    String storeDir = (String) options.valueOf("store-dir");
    ReadOnlyStorageFormat format = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "version", ReadOnlyStorageFormat.READONLY_V2.toString()));
    Cluster cluster = null;
    int nodeId = 0;
    SerializerDefinition sdef = new SerializerDefinition("json", "'string'");
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setKeySerializer(sdef).setValueSerializer(sdef).setRequiredReads(1).setReplicationFactor(1).setRequiredWrites(1).setType("read-only").setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setRoutingPolicy(RoutingTier.CLIENT).build();
    if (options.has("build")) {
        CmdUtils.croakIfMissing(parser, options, "num-values", "value-size");
        numValues = (Integer) options.valueOf("num-values");
        int numChunks = 1;
        if (options.has("num-chunks"))
            numChunks = (Integer) options.valueOf("num-chunks");
        int valueSize = (Integer) options.valueOf("value-size");
        // generate test data
        File temp = null;
        if (options.has("test-gz")) {
            temp = new File((String) options.valueOf("test-gz"));
        } else {
            temp = File.createTempFile("json-data", ".txt.gz", workingDir);
            temp.deleteOnExit();
            System.out.println("Generating test data in " + temp);
            OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(temp));
            Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream), 10 * 1024 * 1024);
            String value = TestUtils.randomLetters(valueSize);
            for (int i = 0; i < numValues; i++) {
                writer.write("\"");
                writer.write(Integer.toString(i));
                writer.write("\" \"");
                writer.write(value);
                writer.write("\"");
                writer.write("\n");
            }
            writer.close();
            writer = null;
        }
        System.out.println("Building store.");
        InputStream inputStream = new GZIPInputStream(new FileInputStream(temp));
        Reader r = new BufferedReader(new InputStreamReader(inputStream), 1 * 1024 * 1024);
        File output = TestUtils.createTempDir(workingDir);
        File tempDir = TestUtils.createTempDir(workingDir);
        cluster = ServerTestUtils.getLocalCluster(1);
        nodeId = 0;
        JsonStoreBuilder builder = new JsonStoreBuilder(new JsonReader(r), cluster, storeDef, new ConsistentRoutingStrategy(cluster, 1), output, tempDir, internalSortSize, 2, numChunks, 64 * 1024, gzipIntermediate);
        builder.build(format);
        // copy to store dir
        File dir = new File(storeDir);
        Utils.rm(dir);
        dir.mkdirs();
        System.out.println("Moving store data from " + output + " to " + dir);
        boolean copyWorked = new File(output, "node-0").renameTo(new File(dir, "version-0"));
        if (!copyWorked)
            Utils.croak("Copy of data from " + output + " to " + dir + " failed.");
    } else {
        CmdUtils.croakIfMissing(parser, options, "cluster-xml", "node-id");
        String clusterXmlPath = (String) options.valueOf("cluster-xml");
        nodeId = (Integer) options.valueOf("node-id");
        File clusterXml = new File(clusterXmlPath);
        if (!clusterXml.exists()) {
            Utils.croak("Cluster.xml does not exist");
        }
        cluster = new ClusterMapper().readCluster(clusterXml);
    }
    final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test", searcher, new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster), nodeId, new File(storeDir), 0);
    final AtomicInteger obsoletes = new AtomicInteger(0);
    final AtomicInteger nullResults = new AtomicInteger(0);
    final AtomicInteger totalResults = new AtomicInteger(0);
    final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
    final Executor executor = Executors.newFixedThreadPool(1);
    // if they have given us a file make a request generator that reads from
    // it, otherwise just generate random values
    final int numVals = numValues;
    Runnable requestGenerator;
    if (inputFile == null) {
        requestGenerator = new Runnable() {

            public void run() {
                System.out.println("Generating random requests.");
                Random random = new Random();
                try {
                    while (true) requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        };
    } else {
        requestGenerator = new Runnable() {

            public void run() {
                try {
                    System.out.println("Using request file to generate requests.");
                    BufferedReader reader = new BufferedReader(new FileReader(inputFile), 1000000);
                    while (true) {
                        String line = reader.readLine();
                        if (line == null)
                            return;
                        requestIds.put(line.trim());
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        };
    }
    executor.execute(requestGenerator);
    final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"), true);
    final AtomicInteger current = new AtomicInteger();
    final int progressIncrement = numRequests / 5;
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                totalResults.incrementAndGet();
                int curr = current.getAndIncrement();
                List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())), null);
                if (curr % progressIncrement == 0)
                    System.out.println(curr);
                if (results.size() == 0)
                    nullResults.incrementAndGet();
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    System.out.println("Running test...");
    readWriteTest.run(numRequests, numThreads);
    System.out.println("Random Access Read Only store Results:");
    System.out.println("Null reads ratio:" + (nullResults.doubleValue()) / totalResults.doubleValue());
    readWriteTest.printStats();
    System.exit(0);
}
Also used : BinarySearchStrategy(voldemort.store.readonly.BinarySearchStrategy) SearchStrategy(voldemort.store.readonly.SearchStrategy) GZIPOutputStream(java.util.zip.GZIPOutputStream) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Random(java.util.Random) GZIPOutputStream(java.util.zip.GZIPOutputStream) ByteArray(voldemort.utils.ByteArray) StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) ReadOnlyStorageEngine(voldemort.store.readonly.ReadOnlyStorageEngine) ReadOnlyStorageFormat(voldemort.store.readonly.ReadOnlyStorageFormat) ClusterMapper(voldemort.xml.ClusterMapper) FileInputStream(java.io.FileInputStream) JsonStoreBuilder(voldemort.store.readonly.JsonStoreBuilder) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FileOutputStream(java.io.FileOutputStream) File(java.io.File) JsonTypeSerializer(voldemort.serialization.json.JsonTypeSerializer) Versioned(voldemort.versioning.Versioned) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Reader(java.io.Reader) JsonReader(voldemort.serialization.json.JsonReader) InputStreamReader(java.io.InputStreamReader) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) OptionParser(joptsimple.OptionParser) BufferedWriter(java.io.BufferedWriter) GZIPInputStream(java.util.zip.GZIPInputStream) Executor(java.util.concurrent.Executor) StoreDefinition(voldemort.store.StoreDefinition) JsonReader(voldemort.serialization.json.JsonReader) ConsistentRoutingStrategy(voldemort.routing.ConsistentRoutingStrategy) FileReader(java.io.FileReader) InputStreamReader(java.io.InputStreamReader) GZIPInputStream(java.util.zip.GZIPInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) Cluster(voldemort.cluster.Cluster) FileNotFoundException(java.io.FileNotFoundException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BufferedReader(java.io.BufferedReader) OutputStreamWriter(java.io.OutputStreamWriter) OptionSet(joptsimple.OptionSet) SerializerDefinition(voldemort.serialization.SerializerDefinition) Writer(java.io.Writer) OutputStreamWriter(java.io.OutputStreamWriter) BufferedWriter(java.io.BufferedWriter)

Example 7 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class MultithreadedStressTest method testGetAndPut.

public void testGetAndPut() throws Exception {
    final AtomicInteger obsoletes = new AtomicInteger(0);
    final CountDownLatch isDone = new CountDownLatch(numberOfRequests);
    for (int i = 0; i < numberOfRequests; i++) {
        final int index = i % numberOfValues;
        service.execute(new Runnable() {

            public void run() {
                boolean done = false;
                while (!done) {
                    try {
                        byte[] key = Integer.toString(index).getBytes();
                        List<Versioned<byte[]>> found = store.get(key, null);
                        if (found.size() > 1) {
                            throw new RuntimeException("Found multiple versions: " + found);
                        } else if (found.size() == 1) {
                            Versioned<byte[]> versioned = found.get(0);
                            byte[] valueBytes = Integer.toString(MultithreadedStressTest.this.value.getAndIncrement()).getBytes();
                            versioned.setObject(valueBytes);
                            store.put(key, versioned, null);
                            done = true;
                        } else if (found.size() == 0) {
                            throw new RuntimeException("No values found!");
                        }
                    } catch (ObsoleteVersionException e) {
                        obsoletes.getAndIncrement();
                    } finally {
                        isDone.countDown();
                    }
                }
            }
        });
    }
    isDone.await();
    System.err.println("Number of obsoletes: " + obsoletes.get());
    System.exit(0);
}
Also used : ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 8 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class AddNodeAction method update.

/**
     * @throws ObsoleteVersionException if a concurrent modification (remove or
     *         another add) has not completed modifying the structure.
     * @throws ArrayIndexOutOfBoundsException if no more ids left to identify
     *         object
     */
@Override
public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
    _storeClient = storeClient;
    VListKey<K> newKey = new VListKey<K>(_key, 0);
    Versioned<Map<String, Object>> firstNodeMap = storeClient.get(newKey.mapValue());
    // adding first node of the list
    if (firstNodeMap == null) {
        Versioned<Map<String, Object>> newNode = new Versioned<Map<String, Object>>((new VListNode<E>(_value, 0, VStack.NULL_ID, VStack.NULL_ID, true)).mapValue());
        // throws ObsoleteVersionException if another process has created a
        // new node already
        storeClient.put(newKey.mapValue(), newNode);
    } else // add to front of list
    {
        Versioned<VListNode<E>> firstNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(firstNodeMap.getValue()), firstNodeMap.getVersion());
        if (!firstNode.getValue().isStable()) {
            throw new ObsoleteVersionException("cannot add when list node is not stable");
        }
        // set stable flag to false
        Map<String, Object> tmpMap = new HashMap<String, Object>(firstNodeMap.getValue());
        tmpMap.put(VListNode.STABLE, false);
        storeClient.put(newKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, firstNodeMap.getVersion()));
        _rollback.put(0, firstNodeMap.getValue());
        int newId;
        int nextId = firstNode.getValue().getNextId();
        newId = (nextId == VStack.NULL_ID) ? 1 : nextId + 1;
        if (newId == Integer.MAX_VALUE)
            throw new ArrayIndexOutOfBoundsException(newId + " out of bounds");
        Versioned<VListNode<E>> nextNode = null;
        VListKey<K> nextKey = new VListKey<K>(_key, nextId);
        if (nextId != VStack.NULL_ID) {
            Versioned<Map<String, Object>> nextNodeMap = storeClient.get(nextKey.mapValue());
            if (nextNodeMap == null)
                throw new ObsoleteVersionException("possible concurrent modification");
            nextNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(nextNodeMap.getValue()), nextNodeMap.getVersion());
            if (!nextNode.getValue().isStable()) {
                throw new ObsoleteVersionException("cannot add when list node is not stable");
            }
            // set stable flag to false
            tmpMap = new HashMap<String, Object>(nextNode.getValue().mapValue());
            tmpMap.put(VListNode.STABLE, false);
            storeClient.put(nextKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, nextNode.getVersion()));
            _rollback.put(nextId, nextNode.getValue().mapValue());
        }
        // insert new node
        Map<String, Object> newNode = (new VListNode<E>(_value, 0, VStack.NULL_ID, newId, true)).mapValue();
        // don't need to specify versioned because node is already "locked"
        storeClient.put(newKey.mapValue(), newNode);
        // move first node to next index
        VListKey<K> firstKey = new VListKey<K>(_key, newId);
        firstNode = new Versioned<VListNode<E>>(new VListNode<E>(firstNode.getValue().getValue(), newId, 0, firstNode.getValue().getNextId(), true));
        // don't need to specify versioned because node is already "locked"
        storeClient.put(firstKey.mapValue(), firstNode.getValue().mapValue());
        // redefine previous pointer on next node
        if (nextNode != null) {
            if (!storeClient.applyUpdate(new UpdateNextNode<K, E>(nextNode, nextKey, newId)))
                throw new ObsoleteVersionException("unable to update node");
        }
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) Map(java.util.Map) HashMap(java.util.HashMap)

Example 9 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class UpdateElementById method update.

@Override
public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
    Versioned<Map<String, Object>> nodeMap = storeClient.get(_key.mapValue());
    if (nodeMap == null)
        throw new IndexOutOfBoundsException("invalid id " + _key.getId());
    Version version = (_version != null) ? _version : nodeMap.getVersion();
    VListNode<E> listNode = VListNode.valueOf(nodeMap.getValue());
    if (!listNode.isStable()) {
        throw new ObsoleteVersionException("node " + _key.getId() + " not stable.");
    }
    _result = listNode.getValue();
    VListNode<E> newNode = new VListNode<E>(_element, listNode.getId(), listNode.getPreviousId(), listNode.getNextId(), true);
    storeClient.put(_key.mapValue(), new Versioned<Map<String, Object>>(newNode.mapValue(), version));
}
Also used : ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) Version(voldemort.versioning.Version) Map(java.util.Map)

Example 10 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class VStack method setById.

/**
     * Put the given value to the appropriate id in the stack, using the version
     * of the current list node identified by that id.
     * 
     * @param id
     * @param element element to set
     * @return element that was replaced by the new element
     * @throws ObsoleteVersionException when an update fails
     */
public E setById(final int id, final E element) {
    VListKey<K> key = new VListKey<K>(_key, id);
    UpdateElementById<K, E> updateElementAction = new UpdateElementById<K, E>(key, element);
    if (!_storeClient.applyUpdate(updateElementAction))
        throw new ObsoleteVersionException("update failed");
    return updateElementAction.getResult();
}
Also used : ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException)

Aggregations

ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)43 ByteArray (voldemort.utils.ByteArray)23 VectorClock (voldemort.versioning.VectorClock)16 Versioned (voldemort.versioning.Versioned)14 Node (voldemort.cluster.Node)11 VoldemortException (voldemort.VoldemortException)10 Test (org.junit.Test)8 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)7 File (java.io.File)6 UnreachableStoreException (voldemort.store.UnreachableStoreException)6 Occurred (voldemort.versioning.Occurred)5 IOException (java.io.IOException)4 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)4 List (java.util.List)4 VoldemortConfig (voldemort.server.VoldemortConfig)4 Date (java.util.Date)3 Random (java.util.Random)3 OptionParser (joptsimple.OptionParser)3 OptionSet (joptsimple.OptionSet)3