Search in sources :

Example 11 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class RequestFileFilter method main.

/**
     * Filter requests specified in a file, generating a new file containing
     * only requests destined for a specific node.
     * 
     * @param args See usage for more information
     * @throws Exception In case of I/O or Voldemort-specific errors
     */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
    parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
    parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
    parser.accepts("string-keys");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    int nodeId = (Integer) options.valueOf("node");
    String storeName = (String) options.valueOf("store-name");
    String bootstrapURL = (String) options.valueOf("url");
    String inputFile = (String) options.valueOf("input");
    String outputFile = (String) options.valueOf("output");
    boolean stringKeys = options.has("string-keys");
    AdminClient adminClient = new AdminClient(bootstrapURL);
    List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : storeDefinitionList) {
        if (storeName.equals(def.getName())) {
            storeDefinition = def;
        }
    }
    if (storeDefinition == null) {
        Utils.croak("No store found with name\"" + storeName + "\"");
    }
    Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
    Node node = null;
    try {
        node = cluster.getNodeById(nodeId);
    } catch (VoldemortException e) {
        Utils.croak("Can't find a node with id " + nodeId);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    try {
        new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}
Also used : RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Node(voldemort.cluster.Node) FileNotFoundException(java.io.FileNotFoundException) Cluster(voldemort.cluster.Cluster) OptionParser(joptsimple.OptionParser) VoldemortException(voldemort.VoldemortException) StoreDefinition(voldemort.store.StoreDefinition) RoutingStrategy(voldemort.routing.RoutingStrategy) OptionSet(joptsimple.OptionSet) AdminClient(voldemort.client.protocol.admin.AdminClient)

Example 12 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}
Also used : HashMap(java.util.HashMap) Node(voldemort.cluster.Node) RoutedStoreFactory(voldemort.store.routed.RoutedStoreFactory) RoutedStore(voldemort.store.routed.RoutedStore) Store(voldemort.store.Store) SleepyStore(voldemort.store.SleepyStore) StoreRepository(voldemort.server.StoreRepository) VoldemortServer(voldemort.server.VoldemortServer) Properties(java.util.Properties) OptionParser(joptsimple.OptionParser) VoldemortException(voldemort.VoldemortException) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) StoreDefinition(voldemort.store.StoreDefinition) BannagePeriodFailureDetector(voldemort.cluster.failuredetector.BannagePeriodFailureDetector) FailureDetector(voldemort.cluster.failuredetector.FailureDetector) ByteArray(voldemort.utils.ByteArray) ClientConfig(voldemort.client.ClientConfig) BannagePeriodFailureDetector(voldemort.cluster.failuredetector.BannagePeriodFailureDetector) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) SocketStoreFactory(voldemort.store.socket.SocketStoreFactory) SleepyStore(voldemort.store.SleepyStore) RoutedStore(voldemort.store.routed.RoutedStore) ClientRequestExecutorPool(voldemort.store.socket.clientrequest.ClientRequestExecutorPool) RoutedStoreConfig(voldemort.store.routed.RoutedStoreConfig) FailureDetectorConfig(voldemort.cluster.failuredetector.FailureDetectorConfig) ExecutorService(java.util.concurrent.ExecutorService) OptionSet(joptsimple.OptionSet) File(java.io.File)

Example 13 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class ReadOnlyStorePerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("requests", "[REQUIRED] number of requests").withRequiredArg().ofType(Integer.class);
    parser.accepts("store-dir", "[REQUIRED] store directory").withRequiredArg().describedAs("directory");
    parser.accepts("cluster-xml", "Path to cluster.xml").withRequiredArg().describedAs("path");
    parser.accepts("node-id", "Id of node").withRequiredArg().ofType(Integer.class).describedAs("node-id");
    parser.accepts("search-strategy", "class of the search strategy to use").withRequiredArg().describedAs("class_name");
    parser.accepts("build", "If present, first build the data");
    parser.accepts("num-values", "The number of values in the store").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts("num-chunks", "The number of chunks per partition").withRequiredArg().describedAs("chunks").ofType(Integer.class);
    parser.accepts("internal-sort-size", "The number of items to sort in memory at a time").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("working-dir", "The directory in which to store temporary data").withRequiredArg().describedAs("dir");
    parser.accepts("gzip", "Compress the intermediate temp files used in building the store");
    parser.accepts("request-file", "file get request ids from").withRequiredArg();
    parser.accepts("version", "Version of read-only store [" + ReadOnlyStorageFormat.READONLY_V0 + "," + ReadOnlyStorageFormat.READONLY_V1 + "," + ReadOnlyStorageFormat.READONLY_V2 + " (default)]").withRequiredArg().describedAs("version");
    parser.accepts("test-gz", "Path to gzip containing data. Works with --build only").withRequiredArg().describedAs("path");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    CmdUtils.croakIfMissing(parser, options, "requests", "store-dir");
    final int numThreads = CmdUtils.valueOf(options, "threads", 10);
    final int numRequests = (Integer) options.valueOf("requests");
    final int internalSortSize = CmdUtils.valueOf(options, "internal-sort-size", 500000);
    int numValues = numRequests;
    final String inputFile = (String) options.valueOf("request-file");
    final String searcherClass = CmdUtils.valueOf(options, "search-strategy", BinarySearchStrategy.class.getName()).trim();
    final boolean gzipIntermediate = options.has("gzip");
    final SearchStrategy searcher = (SearchStrategy) ReflectUtils.callConstructor(ReflectUtils.loadClass(searcherClass));
    final File workingDir = new File(CmdUtils.valueOf(options, "working-dir", System.getProperty("java.io.tmpdir")));
    String storeDir = (String) options.valueOf("store-dir");
    ReadOnlyStorageFormat format = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "version", ReadOnlyStorageFormat.READONLY_V2.toString()));
    Cluster cluster = null;
    int nodeId = 0;
    SerializerDefinition sdef = new SerializerDefinition("json", "'string'");
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setKeySerializer(sdef).setValueSerializer(sdef).setRequiredReads(1).setReplicationFactor(1).setRequiredWrites(1).setType("read-only").setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setRoutingPolicy(RoutingTier.CLIENT).build();
    if (options.has("build")) {
        CmdUtils.croakIfMissing(parser, options, "num-values", "value-size");
        numValues = (Integer) options.valueOf("num-values");
        int numChunks = 1;
        if (options.has("num-chunks"))
            numChunks = (Integer) options.valueOf("num-chunks");
        int valueSize = (Integer) options.valueOf("value-size");
        // generate test data
        File temp = null;
        if (options.has("test-gz")) {
            temp = new File((String) options.valueOf("test-gz"));
        } else {
            temp = File.createTempFile("json-data", ".txt.gz", workingDir);
            temp.deleteOnExit();
            System.out.println("Generating test data in " + temp);
            OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(temp));
            Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream), 10 * 1024 * 1024);
            String value = TestUtils.randomLetters(valueSize);
            for (int i = 0; i < numValues; i++) {
                writer.write("\"");
                writer.write(Integer.toString(i));
                writer.write("\" \"");
                writer.write(value);
                writer.write("\"");
                writer.write("\n");
            }
            writer.close();
            writer = null;
        }
        System.out.println("Building store.");
        InputStream inputStream = new GZIPInputStream(new FileInputStream(temp));
        Reader r = new BufferedReader(new InputStreamReader(inputStream), 1 * 1024 * 1024);
        File output = TestUtils.createTempDir(workingDir);
        File tempDir = TestUtils.createTempDir(workingDir);
        cluster = ServerTestUtils.getLocalCluster(1);
        nodeId = 0;
        JsonStoreBuilder builder = new JsonStoreBuilder(new JsonReader(r), cluster, storeDef, new ConsistentRoutingStrategy(cluster, 1), output, tempDir, internalSortSize, 2, numChunks, 64 * 1024, gzipIntermediate);
        builder.build(format);
        // copy to store dir
        File dir = new File(storeDir);
        Utils.rm(dir);
        dir.mkdirs();
        System.out.println("Moving store data from " + output + " to " + dir);
        boolean copyWorked = new File(output, "node-0").renameTo(new File(dir, "version-0"));
        if (!copyWorked)
            Utils.croak("Copy of data from " + output + " to " + dir + " failed.");
    } else {
        CmdUtils.croakIfMissing(parser, options, "cluster-xml", "node-id");
        String clusterXmlPath = (String) options.valueOf("cluster-xml");
        nodeId = (Integer) options.valueOf("node-id");
        File clusterXml = new File(clusterXmlPath);
        if (!clusterXml.exists()) {
            Utils.croak("Cluster.xml does not exist");
        }
        cluster = new ClusterMapper().readCluster(clusterXml);
    }
    final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test", searcher, new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster), nodeId, new File(storeDir), 0);
    final AtomicInteger obsoletes = new AtomicInteger(0);
    final AtomicInteger nullResults = new AtomicInteger(0);
    final AtomicInteger totalResults = new AtomicInteger(0);
    final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
    final Executor executor = Executors.newFixedThreadPool(1);
    // if they have given us a file make a request generator that reads from
    // it, otherwise just generate random values
    final int numVals = numValues;
    Runnable requestGenerator;
    if (inputFile == null) {
        requestGenerator = new Runnable() {

            public void run() {
                System.out.println("Generating random requests.");
                Random random = new Random();
                try {
                    while (true) requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        };
    } else {
        requestGenerator = new Runnable() {

            public void run() {
                try {
                    System.out.println("Using request file to generate requests.");
                    BufferedReader reader = new BufferedReader(new FileReader(inputFile), 1000000);
                    while (true) {
                        String line = reader.readLine();
                        if (line == null)
                            return;
                        requestIds.put(line.trim());
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        };
    }
    executor.execute(requestGenerator);
    final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"), true);
    final AtomicInteger current = new AtomicInteger();
    final int progressIncrement = numRequests / 5;
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                totalResults.incrementAndGet();
                int curr = current.getAndIncrement();
                List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())), null);
                if (curr % progressIncrement == 0)
                    System.out.println(curr);
                if (results.size() == 0)
                    nullResults.incrementAndGet();
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    System.out.println("Running test...");
    readWriteTest.run(numRequests, numThreads);
    System.out.println("Random Access Read Only store Results:");
    System.out.println("Null reads ratio:" + (nullResults.doubleValue()) / totalResults.doubleValue());
    readWriteTest.printStats();
    System.exit(0);
}
Also used : BinarySearchStrategy(voldemort.store.readonly.BinarySearchStrategy) SearchStrategy(voldemort.store.readonly.SearchStrategy) GZIPOutputStream(java.util.zip.GZIPOutputStream) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Random(java.util.Random) GZIPOutputStream(java.util.zip.GZIPOutputStream) ByteArray(voldemort.utils.ByteArray) StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) ReadOnlyStorageEngine(voldemort.store.readonly.ReadOnlyStorageEngine) ReadOnlyStorageFormat(voldemort.store.readonly.ReadOnlyStorageFormat) ClusterMapper(voldemort.xml.ClusterMapper) FileInputStream(java.io.FileInputStream) JsonStoreBuilder(voldemort.store.readonly.JsonStoreBuilder) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FileOutputStream(java.io.FileOutputStream) File(java.io.File) JsonTypeSerializer(voldemort.serialization.json.JsonTypeSerializer) Versioned(voldemort.versioning.Versioned) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Reader(java.io.Reader) JsonReader(voldemort.serialization.json.JsonReader) InputStreamReader(java.io.InputStreamReader) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) OptionParser(joptsimple.OptionParser) BufferedWriter(java.io.BufferedWriter) GZIPInputStream(java.util.zip.GZIPInputStream) Executor(java.util.concurrent.Executor) StoreDefinition(voldemort.store.StoreDefinition) JsonReader(voldemort.serialization.json.JsonReader) ConsistentRoutingStrategy(voldemort.routing.ConsistentRoutingStrategy) FileReader(java.io.FileReader) InputStreamReader(java.io.InputStreamReader) GZIPInputStream(java.util.zip.GZIPInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) Cluster(voldemort.cluster.Cluster) FileNotFoundException(java.io.FileNotFoundException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BufferedReader(java.io.BufferedReader) OutputStreamWriter(java.io.OutputStreamWriter) OptionSet(joptsimple.OptionSet) SerializerDefinition(voldemort.serialization.SerializerDefinition) Writer(java.io.Writer) OutputStreamWriter(java.io.OutputStreamWriter) BufferedWriter(java.io.BufferedWriter)

Example 14 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AdminServiceBasicTest method testUpdateClusterMetadata.

@Test
public void testUpdateClusterMetadata() {
    Cluster updatedCluster = ServerTestUtils.getLocalCluster(4);
    AdminClient client = getAdminClient();
    for (int i = 0; i < NUM_RUNS; i++) {
        VectorClock existingClock = ((VectorClock) client.metadataMgmtOps.getRemoteCluster(0).getVersion());
        VectorClock clock = existingClock.incremented(0, System.currentTimeMillis());
        client.metadataMgmtOps.updateRemoteCluster(0, updatedCluster, clock);
        assertEquals("Cluster should match", updatedCluster, getVoldemortServer(0).getMetadataStore().getCluster());
        assertEquals("AdminClient.getMetdata() should match", client.metadataMgmtOps.getRemoteCluster(0).getValue(), updatedCluster);
        // version should match
        assertEquals("versions should match as well.", clock, client.metadataMgmtOps.getRemoteCluster(0).getVersion());
    }
}
Also used : VectorClock(voldemort.versioning.VectorClock) Cluster(voldemort.cluster.Cluster) AdminClient(voldemort.client.protocol.admin.AdminClient) Test(org.junit.Test)

Example 15 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AdminServiceBasicTest method testReplicationMappingWithZonePreference.

@Test
public void testReplicationMappingWithZonePreference() {
    List<Zone> zones = ServerTestUtils.getZones(2);
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(0, "localhost", 1, 2, 3, 0, Lists.newArrayList(0, 4, 8)));
    nodes.add(new Node(1, "localhost", 1, 2, 3, 0, Lists.newArrayList(1, 5, 9)));
    nodes.add(new Node(2, "localhost", 1, 2, 3, 1, Lists.newArrayList(2, 6, 10)));
    nodes.add(new Node(3, "localhost", 1, 2, 3, 1, Lists.newArrayList(3, 7, 11)));
    // Test 0 - With rep-factor 1; zone 1
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    Cluster newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // With rep-factor 1; zone 0
    storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // Test 1 - With consistent routing strategy
    storeDef = ServerTestUtils.getStoreDef("consistent", 4, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    // On node 0; zone id 1
    Map<Integer, List<Integer>> replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10));
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // On node 0; zone id 0
    replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        // partitionTuple.put(1, Lists.newArrayList(0, 4, 8));
        // partitionTuple.put(2, Lists.newArrayList(3, 7, 11));
        // partitionTuple.put(3, Lists.newArrayList(2, 6, 10));
        expectedMapping.put(1, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // Test 2 - With zone routing strategy, and zone replication factor 1
    HashMap<Integer, Integer> zoneReplicationFactors = Maps.newHashMap();
    for (int zoneIds = 0; zoneIds < 2; zoneIds++) {
        zoneReplicationFactors.put(zoneIds, 1);
    }
    storeDef = ServerTestUtils.getStoreDef("zone", 2, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    newCluster = new Cluster("multi_zone_cluster", nodes, zones);
    {
        try {
            replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 0);
            fail("Should have thrown an exception since  zoneReplicationFactor is 1");
        } catch (VoldemortException e) {
        }
    }
    {
        // On node 0, zone 1
        replicationMapping = adminClient.replicaOps.getReplicationMapping(0, newCluster, storeDef, 1);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(0, 4, 8, 2, 6, 10));
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 1, zone 1
        replicationMapping = adminClient.replicaOps.getReplicationMapping(1, newCluster, storeDef, 1);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(2, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
}
Also used : HashMap(java.util.HashMap) Zone(voldemort.cluster.Zone) Node(voldemort.cluster.Node) Cluster(voldemort.cluster.Cluster) VoldemortException(voldemort.VoldemortException) StoreDefinition(voldemort.store.StoreDefinition) List(java.util.List) ArrayList(java.util.ArrayList) Test(org.junit.Test)

Aggregations

Cluster (voldemort.cluster.Cluster)197 Test (org.junit.Test)74 StoreDefinition (voldemort.store.StoreDefinition)74 Node (voldemort.cluster.Node)72 ArrayList (java.util.ArrayList)51 HashMap (java.util.HashMap)47 ByteArray (voldemort.utils.ByteArray)33 AdminClient (voldemort.client.protocol.admin.AdminClient)26 ClusterTestUtils (voldemort.ClusterTestUtils)25 VoldemortException (voldemort.VoldemortException)24 List (java.util.List)23 ClusterMapper (voldemort.xml.ClusterMapper)23 File (java.io.File)20 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)18 Zone (voldemort.cluster.Zone)17 Versioned (voldemort.versioning.Versioned)17 Properties (java.util.Properties)16 IOException (java.io.IOException)15 VoldemortServer (voldemort.server.VoldemortServer)15 RoutingStrategyFactory (voldemort.routing.RoutingStrategyFactory)14