Examples with VoldemortConfig voldemort.server.VoldemortConfig used on opensource projects

Example 41 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class BdbCachePartitioningTest method testMinimumSharedCache.

/**
     * Tests that any reservation that will not violate minimum shared cache
     * will fail, during server startup and dynamic updation
     */
@Test
public void testMinimumSharedCache() {
    // total cache size
    int totalCache = 20 * ByteUtils.BYTES_PER_MB;
    // A reserves 10MB
    int shareA = 10 * ByteUtils.BYTES_PER_MB;
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(totalCache);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
    voldemortConfig.setBdbMinimumSharedCache(15 * ByteUtils.BYTES_PER_MB);
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    BdbStorageEngine storeA = null;
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    assertEquals("Reserved cache size not zero", 0, bdbStorage.getReservedCacheSize());
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
        storeA = (BdbStorageEngine) bdbStorage.getStore(defA, TestUtils.makeSingleNodeRoutingStrategy());
        fail("Should have thrown exception since minSharedCache will be violated");
    } catch (StorageInitializationException sie) {
    // should come here.
    }
    // failing operations should not alter reserved cache size
    assertEquals("failure somehow altered the reservedCacheSize", 0, bdbStorage.getReservedCacheSize());
    voldemortConfig.setBdbMinimumSharedCache(10 * ByteUtils.BYTES_PER_MB);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", shareA / ByteUtils.BYTES_PER_MB);
        storeA = (BdbStorageEngine) bdbStorage.getStore(defA, TestUtils.makeSingleNodeRoutingStrategy());
    } catch (StorageInitializationException sie) {
        // should not come here.
        fail("minSharedCache should n't have been violated");
    }
    assertEquals("store A's share does not match up with reserved cache size", shareA, bdbStorage.getReservedCacheSize());
    long reserveCacheSize = bdbStorage.getReservedCacheSize();
    // now, try increasing the reservation dynamically and it should fail
    try {
        StoreDefinition defA = TestUtils.makeStoreDefinition("storeA", 15);
        bdbStorage.update(defA);
        fail("Should have thrown exception since minSharedCache will be violated");
    } catch (StorageInitializationException sie) {
    // should come here.
    }
    assertEquals("failure somehow altered the reservedCacheSize", reserveCacheSize, bdbStorage.getReservedCacheSize());
    if (storeA != null)
        storeA.close();
    bdbStorage.close();
}

Example 42 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class BdbPartitionListIteratorTest method setUp.

@Before
public void setUp() throws Exception {
    bdbMasterDir = TestUtils.createTempDir();
    FileDeleteStrategy.FORCE.delete(bdbMasterDir);
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(10 * 1024 * 1024);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
    voldemortConfig.setBdbPrefixKeysWithPartitionId(true);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    StoreDefinition defA = TestUtils.makeStoreDefinition("storeA");
    store = (BdbStorageEngine) bdbStorage.getStore(defA, (strategy = TestUtils.makeSingleNodeRoutingStrategy()));
    // load some data for non odd partitions, and note down how much data we
    // have for each partition.
    partitionEntries = new HashMap<Integer, Set<String>>();
    int numEntries = 0;
    while (numEntries++ < 10000) {
        String key = "entry_" + numEntries;
        int p = strategy.getMasterPartition(key.getBytes());
        // omit odd partitions
        if (p % 2 == 1)
            continue;
        if (!partitionEntries.containsKey(p))
            partitionEntries.put(p, new HashSet<String>());
        store.put(new ByteArray(key.getBytes()), new Versioned<byte[]>(key.getBytes()), null);
        partitionEntries.get(p).add(key);
    }
}

Example 43 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class BdbBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + BdbBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new BdbStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(30 * 1000 * 1000, 1);
    System.out.println("Bdb write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 44 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class MysqlBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + MysqlBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new MysqlStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(1000, 1);
    System.out.println("MySQl write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 45 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class HdfsFetcher method main.

/**
     * Main method for testing fetching
     */
public static void main(String[] args) throws Exception {
    if (args.length < 1)
        Utils.croak("USAGE: java " + HdfsFetcher.class.getName() + " url [keytab-location kerberos-username hadoop-config-path [destDir]]");
    String url = args[0];
    VoldemortConfig config = new VoldemortConfig(-1, "");
    HdfsFetcher fetcher = new HdfsFetcher(config);
    String destDir = null;
    Long diskQuotaSizeInKB;
    if (args.length >= 4) {
        fetcher.voldemortConfig.setReadOnlyKeytabPath(args[1]);
        fetcher.voldemortConfig.setReadOnlyKerberosUser(args[2]);
        fetcher.voldemortConfig.setHadoopConfigPath(args[3]);
    }
    if (args.length >= 5)
        destDir = args[4];
    if (args.length >= 6)
        diskQuotaSizeInKB = Long.parseLong(args[5]);
    else
        diskQuotaSizeInKB = null;
    // for testing we want to be able to download a single file
    allowFetchingOfSingleFile = true;
    FileSystem fs = HadoopUtils.getHadoopFileSystem(fetcher.voldemortConfig, url);
    Path p = new Path(url);
    FileStatus status = fs.listStatus(p)[0];
    long size = status.getLen();
    long start = System.currentTimeMillis();
    if (destDir == null)
        destDir = System.getProperty("java.io.tmpdir") + File.separator + start;
    File location = fetcher.fetch(url, destDir, null, null, -1, null, diskQuotaSizeInKB);
    double rate = size * Time.MS_PER_SECOND / (double) (System.currentTimeMillis() - start);
    NumberFormat nf = NumberFormat.getInstance();
    nf.setMaximumFractionDigits(2);
    System.out.println("Fetch to " + location + " completed: " + nf.format(rate / (1024.0 * 1024.0)) + " MB/sec.");
    fs.close();
}