Examples with VoldemortException voldemort.VoldemortException used on opensource projects

Example 31 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class ConfigurationStorageEngine method writeVersion.

private void writeVersion(String key, VectorClock version) {
    try {
        File versionFile = new File(getVersionDirectory(), key);
        if (!versionFile.exists() || versionFile.delete()) {
            // write the version file.
            String hexCode = new String(Hex.encodeHex(version.toBytes()));
            FileUtils.writeStringToFile(versionFile, hexCode, "UTF-8");
        }
    } catch (Exception e) {
        throw new VoldemortException("Failed to write Version for Key:" + key, e);
    }
}

Example 32 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class RequestFileFilter method main.

/**
     * Filter requests specified in a file, generating a new file containing
     * only requests destined for a specific node.
     * 
     * @param args See usage for more information
     * @throws Exception In case of I/O or Voldemort-specific errors
     */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
    parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
    parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
    parser.accepts("string-keys");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    int nodeId = (Integer) options.valueOf("node");
    String storeName = (String) options.valueOf("store-name");
    String bootstrapURL = (String) options.valueOf("url");
    String inputFile = (String) options.valueOf("input");
    String outputFile = (String) options.valueOf("output");
    boolean stringKeys = options.has("string-keys");
    AdminClient adminClient = new AdminClient(bootstrapURL);
    List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : storeDefinitionList) {
        if (storeName.equals(def.getName())) {
            storeDefinition = def;
        }
    }
    if (storeDefinition == null) {
        Utils.croak("No store found with name\"" + storeName + "\"");
    }
    Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
    Node node = null;
    try {
        node = cluster.getNodeById(nodeId);
    } catch (VoldemortException e) {
        Utils.croak("Can't find a node with id " + nodeId);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    try {
        new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 33 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}

Example 34 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class Benchmark method initializeStore.

@SuppressWarnings("unchecked")
public void initializeStore(Props benchmarkProps) throws Exception {
    this.numThreads = benchmarkProps.getInt(THREADS, MAX_WORKERS);
    this.numConnectionsPerNode = benchmarkProps.getInt(NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE);
    this.numIterations = benchmarkProps.getInt(ITERATIONS, 1);
    this.statusIntervalSec = benchmarkProps.getInt(INTERVAL, 0);
    this.verbose = benchmarkProps.getBoolean(VERBOSE, false);
    this.verifyRead = benchmarkProps.getBoolean(VERIFY, false);
    this.ignoreNulls = benchmarkProps.getBoolean(IGNORE_NULLS, false);
    int clientZoneId = benchmarkProps.getInt(CLIENT_ZONE_ID, -1);
    if (benchmarkProps.containsKey(URL)) {
        // Remote benchmark
        if (!benchmarkProps.containsKey(STORE_NAME)) {
            throw new VoldemortException("Missing storename");
        }
        String socketUrl = benchmarkProps.getString(URL);
        String storeName = benchmarkProps.getString(STORE_NAME);
        ClientConfig clientConfig = new ClientConfig().setMaxThreads(numThreads).setMaxTotalConnections(numThreads).setMaxConnectionsPerNode(numConnectionsPerNode).setRoutingTimeout(1500, TimeUnit.MILLISECONDS).setSocketTimeout(1500, TimeUnit.MILLISECONDS).setConnectionTimeout(500, TimeUnit.MILLISECONDS).setRequestFormatType(RequestFormatType.VOLDEMORT_V3).setBootstrapUrls(socketUrl);
        if (clientZoneId >= 0) {
            clientConfig.setClientZoneId(clientZoneId);
        }
        SocketStoreClientFactory socketFactory = new SocketStoreClientFactory(clientConfig);
        this.storeClient = socketFactory.getStoreClient(storeName);
        StoreDefinition storeDef = getStoreDefinition(socketFactory, storeName);
        this.keyType = findKeyType(storeDef);
        benchmarkProps.put(Benchmark.KEY_TYPE, this.keyType);
        this.factory = socketFactory;
    } else {
        // Local benchmark
        localMode = true;
        String storageEngineClass = benchmarkProps.getString(STORAGE_CONFIGURATION_CLASS);
        this.keyType = benchmarkProps.getString(KEY_TYPE, STRING_KEY_TYPE);
        Serializer serializer = findKeyType(this.keyType);
        Store<Object, Object, Object> store = null;
        VoldemortConfig voldemortConfig;
        if (benchmarkProps.containsKey(LOCAL_SERVER_PROPERTIES)) {
            File homeDir = TestUtils.createTempDir();
            File configDir = new File(homeDir, "config");
            configDir.mkdir();
            FileUtils.copyFile(new File(benchmarkProps.get(LOCAL_SERVER_PROPERTIES)), new File(configDir, "server.properties"));
            voldemortConfig = VoldemortConfig.loadFromVoldemortHome(homeDir.getAbsolutePath());
        } else {
            voldemortConfig = ServerTestUtils.getVoldemortConfig();
        }
        StorageConfiguration conf = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { voldemortConfig });
        StorageEngine<ByteArray, byte[], byte[]> engine = conf.getStore(TestUtils.makeStoreDefinition(DUMMY_DB), TestUtils.makeSingleNodeRoutingStrategy());
        if (conf.getType().compareTo(ViewStorageConfiguration.TYPE_NAME) == 0) {
            engine = new ViewStorageEngine(STORE_NAME, engine, new StringSerializer(), new StringSerializer(), serializer, new StringSerializer(), null, BenchmarkViews.loadTransformation(benchmarkProps.getString(VIEW_CLASS).trim()));
        }
        store = SerializingStore.wrap(engine, serializer, new StringSerializer(), new IdentitySerializer());
        this.factory = new StaticStoreClientFactory(store);
        this.storeClient = factory.getStoreClient(store.getName());
    }
    this.storeInitialized = true;
}

Example 35 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class AdminServiceBasicTest method testReplicationMappingWithZonePreference.

@Test
public void testReplicationMappingWithZonePreference() {
    List<Zone> zones = ServerTestUtils.getZones(2);
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(0, "localhost", 1, 2, 3, 0, Lists.newArrayList(0, 4, 8)));
    nodes.add(new Node(1, "localhost", 1, 2, 3, 0, Lists.newArrayList(1, 5, 9)));
    nodes.add(new Node(2, "localhost", 1, 2, 3, 1, Lists.newArrayList(2, 6, 10)));
    nodes.add(new Node(3, "localhost", 1, 2, 3, 1, Lists.newArrayList(3, 7, 11)));
    // Test 0 - With rep-factor 1; zone 1
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    Cluster newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // With rep-factor 1; zone 0
    storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // Test 1 - With consistent routing strategy
    storeDef = ServerTestUtils.getStoreDef("consistent", 4, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    // On node 0; zone id 1
    Map<Integer, List<Integer>> replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10));
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // On node 0; zone id 0
    replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        // partitionTuple.put(1, Lists.newArrayList(0, 4, 8));
        // partitionTuple.put(2, Lists.newArrayList(3, 7, 11));
        // partitionTuple.put(3, Lists.newArrayList(2, 6, 10));
        expectedMapping.put(1, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // Test 2 - With zone routing strategy, and zone replication factor 1
    HashMap<Integer, Integer> zoneReplicationFactors = Maps.newHashMap();
    for (int zoneIds = 0; zoneIds < 2; zoneIds++) {
        zoneReplicationFactors.put(zoneIds, 1);
    }
    storeDef = ServerTestUtils.getStoreDef("zone", 2, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    newCluster = new Cluster("multi_zone_cluster", nodes, zones);
    {
        try {
            replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
            fail("Should have thrown an exception since  zoneReplicationFactor is 1");
        } catch (VoldemortException e) {
        }
    }
    {
        // On node 0, zone 1
        replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 2, 6, 10));
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 1, zone 1
        replicationMapping = adminClient.replicaOps.getReplicationMapping(1, newCluster, storeDef, 1);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
}