Examples with VoldemortConfig voldemort.server.VoldemortConfig used on opensource projects

Example 21 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class BdbSplitStorageEngineTest method testMultipleEnvironment.

@Test
public void testMultipleEnvironment() {
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(1 * 1024 * 1024);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    BdbStorageEngine storeA = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeA"), TestUtils.makeSingleNodeRoutingStrategy());
    BdbStorageEngine storeB = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeB"), TestUtils.makeSingleNodeRoutingStrategy());
    storeA.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.close();
    storeB.close();
    assertEquals("StoreA BDB file should exists.", true, (new File(bdbMasterDir + "/" + "storeA").exists()));
    assertEquals("StoreB BDB file should  exists.", true, (new File(bdbMasterDir + "/" + "storeB").exists()));
}

Example 22 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class ExportBDBToTextDump method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    // bdb_folder output_folder
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String outputFolderPath = (String) options.valueOf("output");
    File storeBdbFolder = new File(storeBdbFolderPath);
    File outputFolder = new File(outputFolderPath);
    final String storeName = storeBdbFolder.getName();
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition storeDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(storeDef, null);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    BufferedWriter splitFileWriter = null;
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries = engine.entries();
    while (entries.hasNext()) {
        if (splitFileWriter == null) {
            long splitId = count / SPLIT_SIZE;
            File splitFile = new File(outputFolder, makeSplitFileName(splitId));
            splitFileWriter = new BufferedWriter(new FileWriter(splitFile), WRITER_BUFFER_SIZE);
        }
        Pair<ByteArray, Versioned<byte[]>> pair = entries.next();
        String line = makeLine(pair);
        splitFileWriter.write(line);
        if ((count + 1) % SPLIT_SIZE == 0) {
            splitFileWriter.close();
            splitFileWriter = null;
        }
        count++;
        final Long countObject = count;
        Boolean reported = rp.tryReport(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                System.out.print(String.format("Exported %15d entries", countObject));
                return true;
            }
        });
        if (reported != null) {
            System.out.println(String.format("; Speed: %8d/s", (count - lastCount) / (reportIntervalMs / 1000)));
            lastCount = count;
        }
    }
    entries.close();
    if (splitFileWriter != null) {
        splitFileWriter.close();
    }
    System.out.println(String.format("Finished exporting %d entries", count));
}

Example 23 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class ImportTextDumpToBDB method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    String inputPath = (String) options.valueOf("input");
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String clusterXmlPath = (String) options.valueOf("cluster-xml");
    String storesXmlPath = (String) options.valueOf("stores-xml");
    Integer nodeId = (Integer) options.valueOf("node-id");
    File input = new File(inputPath);
    List<File> dataFiles = new ArrayList<File>();
    if (input.isDirectory()) {
        File[] files = input.listFiles();
        if (files != null)
            Collections.addAll(dataFiles, files);
    } else if (input.isFile()) {
        dataFiles.add(input);
    } else {
        System.err.println(inputPath + "is not file or directory");
    }
    File storeBdbFolder = new File(storeBdbFolderPath);
    final String storeName = storeBdbFolder.getName();
    Cluster cluster = new ClusterMapper().readCluster(new File(clusterXmlPath));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlPath));
    StoreDefinition storeDef = null;
    for (StoreDefinition sd : storeDefs) {
        if (sd.getName() != null && sd.getName().equals(storeName)) {
            storeDef = sd;
        }
    }
    if (storeDef == null) {
        throw new VoldemortException("StoreNotfound: " + storeName);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition mockStoreDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(mockStoreDef, routingStrategy);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    long lastInserted = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    long inserted = 0;
    for (File f : dataFiles) {
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(f), READER_BUFFER_SIZE);
            engine.beginBatchModifications();
            while (true) {
                String line = bufferedReader.readLine();
                if (line == null) {
                    break;
                }
                Pair<ByteArray, Versioned<byte[]>> entry;
                try {
                    entry = lineToEntry(line);
                } catch (Exception e) {
                    System.err.println("Skipping line: " + line);
                    e.printStackTrace();
                    continue;
                }
                ByteArray key = entry.getFirst();
                List<Node> nodeList = routingStrategy.routeRequest(key.get());
                for (Node node : nodeList) {
                    if (nodeId == node.getId()) {
                        try {
                            engine.put(key, entry.getSecond(), null);
                            inserted++;
                        } catch (ObsoleteVersionException e) {
                            e.printStackTrace();
                        }
                        break;
                    }
                }
                count++;
                final Long countObject = count;
                final Long insertedObject = inserted;
                Boolean reported = rp.tryReport(new Callable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                        System.out.print(String.format("Imported %15d entries; Inserted %15d entries", countObject, insertedObject));
                        return true;
                    }
                });
                if (reported != null) {
                    long importSpeed = (count - lastCount) / (reportIntervalMs / 1000);
                    long insertSpeed = (inserted - lastInserted) / (reportIntervalMs / 1000);
                    System.out.println(String.format("; ImportSpeed: %8d/s; InsertSpeed: %8d/s ", importSpeed, insertSpeed));
                    lastCount = count;
                    lastInserted = inserted;
                }
            }
            bufferedReader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            engine.endBatchModifications();
        }
    }
    engine.close();
    System.out.println(String.format("Finished importing %d entries (%d inserted, rest discarded)", count, inserted));
}

Example 24 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class ValidateNodeIdCLI method main.

public static void main(String[] args) throws IOException {
    OptionParser parser = null;
    OptionSet options = null;
    try {
        parser = setupParser();
        options = parser.parse(args);
    } catch (OptionException oe) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Exception when parsing arguments : " + oe.getMessage());
        return;
    }
    /* validate options */
    if (options.has("help")) {
        printUsage();
        return;
    }
    if (!options.hasArgument("id") || !options.hasArgument("path")) {
        printUsageAndDie("Missing a required argument.");
        return;
    }
    String id = (String) options.valueOf("id");
    int expectedNodeId = Integer.parseInt(id);
    String clusterPaths = getFilePath(options, "path");
    String[] allPaths = clusterPaths.split(",");
    boolean isMatch = false;
    for (String path : allPaths) {
        path = path.replace("~", System.getProperty("user.home"));
        File filePath = new File(path);
        if (filePath.exists()) {
            isMatch = true;
            Cluster cluster = getCluster(filePath);
            Properties properties = new Properties();
            properties.setProperty(VoldemortConfig.ENABLE_NODE_ID_DETECTION, Boolean.toString(true));
            properties.setProperty(VoldemortConfig.VOLDEMORT_HOME, getTempDirPath());
            VoldemortConfig config = new VoldemortConfig(properties);
            int actualNodeId = VoldemortServer.getNodeId(config, cluster);
            if (actualNodeId != expectedNodeId) {
                throw new VoldemortApplicationException("Mismatch detected. Computed Node Id " + actualNodeId + " Expected " + expectedNodeId);
            } else {
                System.out.println("Expected and Computed node Id matched " + expectedNodeId);
            }
            config.setNodeId(actualNodeId);
            VoldemortServer.validateNodeId(config, cluster);
            System.out.println("Validation of node Id passed");
        }
    }
    if (!isMatch) {
        throw new VoldemortApplicationException("None of the paths matched the cluster.xml");
    }
}

Example 25 with VoldemortConfig

use of voldemort.server.VoldemortConfig in project voldemort by voldemort.

the class BlockingSlopPusherTest method setUp.

@Override
protected void setUp() throws Exception {
    Cluster cluster = makeCluster(3);
    repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
    MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
    Props props = new Props();
    props.put("node.id", 0);
    props.put("voldemort.home", "/");
    pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}