Examples with VectorClock voldemort.versioning.VectorClock used on opensource projects

Example 16 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class QuotaUtils method setQuota.

public static void setQuota(String storeName, QuotaType type, StoreRepository repository, Set<Integer> nodeIds, long lquota) {
    FileBackedCachingStorageEngine quotaStore = getQuotaStore(repository);
    String quotaKey = makeQuotaKey(storeName, QuotaType.STORAGE_SPACE);
    ByteArray keyArray = convertToByteArray(quotaKey);
    List<Versioned<byte[]>> existingValue = quotaStore.get(keyArray, null);
    String quotaValue = Long.toString(lquota);
    ByteArray valueArray = convertToByteArray(quotaValue);
    VectorClock newClock = VectorClockUtils.makeClockWithCurrentTime(nodeIds);
    Versioned<byte[]> newValue = new Versioned<byte[]>(valueArray.get(), newClock);
    quotaStore.put(keyArray, newValue, null);
}

Example 17 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class StorageEnginePerformanceTest method main.

public static void main(String[] args) throws Exception {
    try {
        OptionParser parser = new OptionParser();
        parser.accepts("help", "print usage information");
        parser.accepts("requests", "[REQUIRED] number of requests to execute").withRequiredArg().ofType(Integer.class);
        parser.accepts("num-values", "[REQUIRED] number of values in the store").withRequiredArg().ofType(Integer.class);
        parser.accepts("data-dir", "Data directory for storage data").withRequiredArg().describedAs("directory");
        parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
        parser.accepts("storage-configuration-class", "[REQUIRED] class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class_name");
        parser.accepts("props", "Properties file with configuration for the engine").withRequiredArg().describedAs("config.properties");
        parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
        parser.accepts("cache-width", "Percentage of requests to save as possible re-requests").withRequiredArg().describedAs("width").ofType(Integer.class);
        parser.accepts("cache-hit-ratio", "Percentage of requests coming from the last cache-width requests").withRequiredArg().describedAs("ratio").ofType(Double.class);
        parser.accepts("clean-up", "Delete data directory when done.");
        OptionSet options = parser.parse(args);
        if (options.has("help")) {
            parser.printHelpOn(System.out);
            System.exit(0);
        }
        CmdUtils.croakIfMissing(parser, options, "requests");
        final int numThreads = CmdUtils.valueOf(options, "threads", 10);
        final int numRequests = (Integer) options.valueOf("requests");
        final int numValues = (Integer) options.valueOf("num-values");
        final int valueSize = CmdUtils.valueOf(options, "value-size", 1024);
        final int cacheWidth = CmdUtils.valueOf(options, "cache-width", 100000);
        final double cacheHitRatio = CmdUtils.valueOf(options, "cache-hit-ratio", 0.5);
        final String propsFile = (String) options.valueOf("props");
        final boolean cleanUp = options.has("clean-up");
        final String storageEngineClass = CmdUtils.valueOf(options, "storage-configuration-class", BdbStorageConfiguration.class.getName()).trim();
        File dataDir = null;
        if (options.has("data-dir"))
            dataDir = new File((String) options.valueOf("data-dir"));
        else
            dataDir = TestUtils.createTempDir();
        System.out.println("Data dir: " + dataDir);
        // create the storage engine
        Props props = new Props();
        if (propsFile != null)
            props = new Props(new File(propsFile));
        props.put("node.id", 0);
        props.put("data.directory", dataDir.getAbsolutePath());
        props.put("voldemort.home", System.getProperty("user.dir"));
        VoldemortConfig config = new VoldemortConfig(props);
        StorageConfiguration storageConfig = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { config });
        StorageEngine<ByteArray, byte[], byte[]> engine = storageConfig.getStore(TestUtils.makeStoreDefinition("test"), TestUtils.makeSingleNodeRoutingStrategy());
        @SuppressWarnings("unchecked") final Store<String, byte[], byte[]> store = new SerializingStore(engine, new StringSerializer(), new IdentitySerializer(), null);
        final byte[] value = new byte[valueSize];
        new Random().nextBytes(value);
        // initialize test data
        for (int i = 0; i < numValues; i++) store.put(Integer.toString(i), Versioned.value(value), null);
        // initialize cache lookback data
        int[] recents = new int[cacheWidth];
        System.out.println("Write test:");
        CachedPerformanceTest writeTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                try {
                    String key = Integer.toString(index);
                    List<Versioned<byte[]>> vs = store.get(key, null);
                    VectorClock version;
                    if (vs.size() == 0)
                        version = new VectorClock();
                    else
                        version = (VectorClock) vs.get(0).getVersion();
                    version.incrementVersion(0, 847584375);
                    store.put(key, Versioned.value(value, version), null);
                } catch (ObsoleteVersionException e) {
                // do nothing
                } catch (RuntimeException e) {
                    e.printStackTrace();
                    throw e;
                }
            }
        }, recents, numValues, cacheHitRatio);
        writeTest.run(numRequests, numThreads);
        writeTest.printStats();
        System.out.println();
        System.out.println("Read test:");
        CachedPerformanceTest readTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                store.get(Integer.toString(index), null);
            }
        }, recents, numValues, cacheHitRatio);
        readTest.run(numRequests, numThreads);
        readTest.printStats();
        if (cleanUp)
            Utils.rm(dataDir);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }
}

Example 18 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class AdminServiceBasicTest method testUpdateClusterMetadata.

@Test
public void testUpdateClusterMetadata() {
    Cluster updatedCluster = ServerTestUtils.getLocalCluster(4);
    AdminClient client = getAdminClient();
    for (int i = 0; i < NUM_RUNS; i++) {
        VectorClock existingClock = ((VectorClock) client.metadataMgmtOps.getRemoteCluster(0).getVersion());
        VectorClock clock = existingClock.incremented(0, System.currentTimeMillis());
        client.metadataMgmtOps.updateRemoteCluster(0, updatedCluster, clock);
        assertEquals("Cluster should match", updatedCluster, getVoldemortServer(0).getMetadataStore().getCluster());
        assertEquals("AdminClient.getMetdata() should match", client.metadataMgmtOps.getRemoteCluster(0).getValue(), updatedCluster);
        // version should match
        assertEquals("versions should match as well.", clock, client.metadataMgmtOps.getRemoteCluster(0).getVersion());
    }
}

Example 19 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class StorageService method initializeMetadataVersions.

protected void initializeMetadataVersions(List<StoreDefinition> storeDefs) {
    Store<ByteArray, byte[], byte[]> versionStore = storeRepository.getLocalStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    Properties props = new Properties();
    try {
        boolean isPropertyAdded = false;
        ByteArray metadataVersionsKey = new ByteArray(SystemStoreConstants.VERSIONS_METADATA_KEY.getBytes());
        List<Versioned<byte[]>> versionList = versionStore.get(metadataVersionsKey, null);
        VectorClock newClock = null;
        if (versionList != null && versionList.size() > 0) {
            byte[] versionsByteArray = versionList.get(0).getValue();
            if (versionsByteArray != null) {
                props.load(new ByteArrayInputStream(versionsByteArray));
            }
            newClock = (VectorClock) versionList.get(0).getVersion();
            newClock = newClock.incremented(0, System.currentTimeMillis());
        } else {
            newClock = new VectorClock();
        }
        // Check if version exists for cluster.xml
        if (!props.containsKey(SystemStoreConstants.CLUSTER_VERSION_KEY)) {
            props.setProperty(SystemStoreConstants.CLUSTER_VERSION_KEY, "0");
            isPropertyAdded = true;
        }
        // Check if version exists for stores.xml
        if (!props.containsKey(SystemStoreConstants.STORES_VERSION_KEY)) {
            props.setProperty(SystemStoreConstants.STORES_VERSION_KEY, "0");
            isPropertyAdded = true;
        }
        // Check if version exists for each store
        for (StoreDefinition def : storeDefs) {
            if (!props.containsKey(def.getName())) {
                props.setProperty(def.getName(), "0");
                isPropertyAdded = true;
            }
        }
        if (isPropertyAdded) {
            StringBuilder finalVersionList = new StringBuilder();
            for (String propName : props.stringPropertyNames()) {
                finalVersionList.append(propName + "=" + props.getProperty(propName) + "\n");
            }
            versionStore.put(metadataVersionsKey, new Versioned<byte[]>(finalVersionList.toString().getBytes(), newClock), null);
        }
    } catch (Exception e) {
        logger.error("Error while intializing metadata versions ", e);
    }
}

Example 20 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class StorageService method calculateStats.

private DataSetStats calculateStats(StorageEngine<ByteArray, byte[], byte[]> store) {
    DataSetStats stats = new DataSetStats();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = store.entries();
    try {
        int count = 0;
        while (iter.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> pair = iter.next();
            VectorClock clock = (VectorClock) pair.getSecond().getVersion();
            stats.countEntry(pair.getFirst().length(), pair.getSecond().getValue().length + clock.sizeInBytes());
            if (count % 10000 == 0)
                logger.debug("Processing key " + count);
            count++;
        }
    } finally {
        iter.close();
    }
    return stats;
}