Examples with Props voldemort.utils.Props used on opensource projects

Example 26 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class BdbBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + BdbBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new BdbStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(30 * 1000 * 1000, 1);
    System.out.println("Bdb write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 27 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class MysqlBuildPerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    if (args.length != 3)
        Utils.croak("USAGE: java " + MysqlBuildPerformanceTest.class.getName() + "serverPropsFile storeName jsonSequenceDataFile");
    String serverPropsFile = args[0];
    String storeName = args[1];
    String jsonDataFile = args[2];
    final Store<ByteArray, byte[], byte[]> store = new MysqlStorageConfiguration(new VoldemortConfig(new Props(new File(serverPropsFile)))).getStore(TestUtils.makeStoreDefinition(storeName), TestUtils.makeSingleNodeRoutingStrategy());
    final AtomicInteger obsoletes = new AtomicInteger(0);
    Path jsonFilePath = new Path(jsonDataFile);
    FileStatus jsonFileStatus = jsonFilePath.getFileSystem(new Configuration()).listStatus(jsonFilePath)[0];
    final SequenceFileRecordReader<BytesWritable, BytesWritable> reader = new SequenceFileRecordReader<BytesWritable, BytesWritable>(new Configuration(), new FileSplit(jsonFilePath, 0, jsonFileStatus.getLen(), (String[]) null));
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                BytesWritable key = new BytesWritable();
                BytesWritable value = new BytesWritable();
                reader.next(key, value);
                store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())), Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    readWriteTest.run(1000, 1);
    System.out.println("MySQl write throuhput with one thread:");
    readWriteTest.printStats();
}

Example 28 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class BuildTestStore method run.

public int run(String[] args) throws Exception {
    if (args.length != 5)
        Utils.croak("Expected arguments store_name config_dir temp_dir input_path output_path");
    String storeName = args[0];
    String configDir = args[1];
    String tempDir = args[2];
    String inputDir = args[3];
    String outputDir = args[4];
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(configDir, "stores.xml"));
    StoreDefinition def = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        def = d;
    Cluster cluster = new ClusterMapper().readCluster(new File(configDir, "cluster.xml"));
    JobConf config = new JobConf();
    String jobName = "test-store-builder";
    config.set("mapred.job.name", jobName);
    HadoopStoreBuilder builder = new HadoopStoreBuilder(jobName, new Props(), config, BuildTestStoreMapper.class, SequenceFileInputFormat.class, cluster, def, new Path(tempDir), new Path(outputDir), new Path(inputDir), CheckSum.CheckSumType.NONE, false, false, (long) (1.5 * 1024 * 1024 * 1024), false, null, false);
    builder.build();
    return 0;
}

Example 29 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class DataCleanupJobTest method setUp.

@Before
public void setUp() throws Exception {
    time = new MockTime();
    storeDir = TestUtils.createTempDir();
    FileDeleteStrategy.FORCE.delete(storeDir);
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(1024 * 1024);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(storeDir.toURI().getPath());
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    StoreDefinition storeDef = getStoreDef(START_RETENTION);
    engine = bdbStorage.getStore(storeDef, TestUtils.makeSingleNodeRoutingStrategy());
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(0, "test-host", 1234, 1235, 1236, Arrays.asList(0)));
    Cluster cluster = new Cluster("cluster", nodes);
    StoreRepository repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, engine);
    metadataStore = ServerTestUtils.createMetadataStore(cluster, Arrays.asList(storeDef));
}

Example 30 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class VoldemortConfig method getDynamicDefaults.

/**
     * This function returns a set of default configs which cannot be defined statically,
     * because they (at least potentially) depend on the config values provided by the user.
     */
private Props getDynamicDefaults(Props userSuppliedConfig) {
    // Combined set of configs made up of user supplied configs first, while falling back
    // on statically defined defaults when the value is missing from the user supplied ones.
    Props combinedConfigs = new Props(userSuppliedConfig, defaultConfig);
    // Set of dynamic configs which depend on the combined configs in order to be determined.
    Props dynamicDefaults = new Props();
    initializeNodeId(combinedConfigs, dynamicDefaults);
    // Define various paths
    String defaultDataDirectory = combinedConfigs.getString(VOLDEMORT_HOME) + File.separator + "data";
    String dataDirectory = combinedConfigs.getString(DATA_DIRECTORY, defaultDataDirectory);
    dynamicDefaults.put(DATA_DIRECTORY, dataDirectory);
    dynamicDefaults.put(BDB_DATA_DIRECTORY, dataDirectory + File.separator + "bdb");
    dynamicDefaults.put(READONLY_DATA_DIRECTORY, dataDirectory + File.separator + "read-only");
    dynamicDefaults.put(ROCKSDB_DATA_DIR, dataDirectory + File.separator + "rocksdb");
    String metadataDirectory = combinedConfigs.getString(VOLDEMORT_HOME) + File.separator + "config";
    dynamicDefaults.put(METADATA_DIRECTORY, metadataDirectory);
    dynamicDefaults.put(READONLY_KEYTAB_PATH, metadataDirectory + File.separator + "voldemrt.headless.keytab");
    dynamicDefaults.put(READONLY_HADOOP_CONFIG_PATH, metadataDirectory + File.separator + "hadoop-conf");
    // Other "transitive" config values.
    dynamicDefaults.put(CORE_THREADS, Math.max(1, combinedConfigs.getInt(MAX_THREADS) / 2));
    dynamicDefaults.put(ADMIN_CORE_THREADS, Math.max(1, combinedConfigs.getInt(ADMIN_MAX_THREADS) / 2));
    dynamicDefaults.put(CLIENT_ROUTING_GET_TIMEOUT_MS, combinedConfigs.getInt(CLIENT_ROUTING_TIMEOUT_MS));
    dynamicDefaults.put(CLIENT_ROUTING_GETALL_TIMEOUT_MS, combinedConfigs.getInt(CLIENT_ROUTING_TIMEOUT_MS));
    int clientRoutingPutTimeoutMs = combinedConfigs.getInt(CLIENT_ROUTING_TIMEOUT_MS);
    dynamicDefaults.put(CLIENT_ROUTING_PUT_TIMEOUT_MS, clientRoutingPutTimeoutMs);
    dynamicDefaults.put(CLIENT_ROUTING_GETVERSIONS_TIMEOUT_MS, combinedConfigs.getInt(CLIENT_ROUTING_PUT_TIMEOUT_MS, clientRoutingPutTimeoutMs));
    dynamicDefaults.put(CLIENT_ROUTING_DELETE_TIMEOUT_MS, combinedConfigs.getInt(CLIENT_ROUTING_TIMEOUT_MS));
    dynamicDefaults.put(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD, combinedConfigs.getInt(SOCKET_TIMEOUT_MS));
    dynamicDefaults.put(REST_SERVICE_STORAGE_THREAD_POOL_QUEUE_SIZE, combinedConfigs.getInt(NUM_REST_SERVICE_STORAGE_THREADS));
    // "failuredetector.bannage.period" so if we have the old one, migrate it over.
    if (userSuppliedConfig.containsKey(CLIENT_NODE_BANNAGE_MS) && !userSuppliedConfig.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD)) {
        dynamicDefaults.put(FAILUREDETECTOR_BANNAGE_PERIOD, userSuppliedConfig.getInt(CLIENT_NODE_BANNAGE_MS));
    }
    return dynamicDefaults;
}