Examples with Cluster voldemort.cluster.Cluster used on opensource projects

Example 21 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testRWRebalanceWithReplication.

@Test(timeout = 600000)
public void testRWRebalanceWithReplication() throws Exception {
    logger.info("Starting testRWRebalanceWithReplication");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        // start servers 0 , 1 only
        List<Integer> serverList = Arrays.asList(0, 1);
        currentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, null);
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
            checkConsistentMetadata(finalCluster, serverList);
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testRWRebalanceWithReplication ", ae);
        throw ae;
    }
}

Example 22 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testRORWRebalance.

@Test(timeout = 600000)
public void testRORWRebalance() throws Exception {
    logger.info("Starting testRORWRebalance");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6, 7, 8 }, {} });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        // start servers 0 , 1 only
        List<Integer> serverList = Arrays.asList(0, 1);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "50");
        currentCluster = startServers(currentCluster, storeDefFileWithoutReplication, serverList, configProps);
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
        try {
            // Populate the two stores
            populateData(currentCluster, roStoreDefWithoutReplication, rebalanceKit.controller.getAdminClient(), true);
            populateData(currentCluster, rwStoreDefWithoutReplication, rebalanceKit.controller.getAdminClient(), false);
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(1));
            checkConsistentMetadata(finalCluster, serverList);
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testRORWRebalance ", ae);
        throw ae;
    }
}

Example 23 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testRORebalanceWithReplication.

@Test(timeout = 600000)
public void testRORebalanceWithReplication() throws Exception {
    logger.info("Starting testRORebalanceWithReplication");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 2, 4, 6 }, { 1, 3, 5, 7 }, {} });
        /*  pre-rebalance partitioning
                0 -> 0, 2, 4, 6
                1 -> 1, 3, 5, 7
                pre-rebalance routing:
                partition -> nodes (ordered)
                0 -> 0, 1
                1 -> 1, 0
                2 -> 0, 1
                3 -> 1, 0
                4 -> 0, 1
                5 -> 1, 0
                6 -> 0, 1
                7 -> 1, 0
            */
        Cluster finalCluster = currentCluster;
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 0, Lists.newArrayList(0, 2, 4, 7));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 1, Lists.newArrayList(1, 3, 5, 6));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 2, Lists.newArrayList(2, 3));
        /*
            post-rebalance partitioning:
                nodes -> partitions
                0 -> 0, 4, 7
                1 -> 1, 5, 6
                2 -> 2, 3
            post-rebalance routing:
                partition -> nodes (ordered)
                0 -> 0, 1
                1 -> 1, 2  (changed from 1, 0)
                2 -> 2, 0  (changed from 0, 1)
                3 -> 2, 0  (changed from 1, 0)
                4 -> 0, 1
                5 -> 1, 0
                6 -> 1, 0  (changed from 0, 1)
                7 -> 0, 1  (changed from 1, 0)
            supposed non-optimized moves:
                partition: node(reptype) -> node(reptype)
                1: 0(second) -> 2(second)
                2: 0(first)  -> 2(first)
                2: 1(second) -> 0(second)
                3: 1(first)  -> 2(first)
                6: 0(first)  -> 1(first)
                6: 1(second) -> 0(second)
                7: 1(first)  -> 0(first)
                7: 0(second) -> 1(second)
            supposed optimized moves:
                partition: node(reptype) -> node(reptype)
                1: 0(second) -> 2(second)
                2: 0(first)  -> 0(second)  * optimized, no cross-node data copy
                2: 1(second) -> 2(first)   * optimized
                3: 1(first)  -> 2(first)
                6: 0(first)  -> 0(second)  * optimized, no cross-node data copy
                6: 1(second) -> 1(first)   * optimized, no cross-node data copy
                7: 0(second) -> 0(first)   * optimized, no cross-node data copy
                7: 1(first)  -> 1(second)  * optimized, no cross-node data copy
            */
        // start servers 0 , 1, 2 only
        List<Integer> serverList = Arrays.asList(0, 1, 2);
        // If this test fails, consider increasing the number of admin
        // threads.
        // In particular, if this test fails by RejectedExecutionHandler in
        // SocketServer.java fires with an error message like the following:
        // "[18:46:32,994
        // voldemort.server.socket.SocketServer[admin-server]]
        // ERROR Too many open connections, 20 of 20 threads in use, denying
        // connection from /127.0.0.1:43756 [Thread-552]". Note, this issues
        // seems to only affect ThreadPoolBasedNonblockingStoreImpl tests
        // rather
        // than Nio-based tests.
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        currentCluster = startServers(currentCluster, roStoreDefFileWithReplication, serverList, configProps);
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
        try {
            populateData(currentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2));
            checkConsistentMetadata(finalCluster, serverList);
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testRORebalanceWithReplication ", ae);
        throw ae;
    }
}

Example 24 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testServerSideRouting.

@Test(timeout = 600000)
public void testServerSideRouting() throws Exception {
    logger.info("Starting testServerSideRouting");
    try {
        final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        final List<Integer> serverList = Arrays.asList(0, 1);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "50");
        configProps.put("enable.server.routing", "true");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingToken = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        // Populate the two stores
        populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
        populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
        Node node = updatedCurrentCluster.getNodeById(1);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRW = getSocketStore(testStoreNameRW, node.getHost(), node.getSocketPort(), true);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRO = getSocketStore(testStoreNameRO, node.getHost(), node.getSocketPort(), true);
        final CountDownLatch latch = new CountDownLatch(1);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    List<String> keys = new ArrayList<String>(testEntries.keySet());
                    while (!rebalancingToken.get()) {
                        // should always able to get values.
                        int index = (int) (Math.random() * keys.size());
                        // should get a valid value
                        try {
                            List<Versioned<byte[]>> values = serverSideRoutingStoreRW.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                            values = serverSideRoutingStoreRO.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                        } catch (UnreachableStoreException e) {
                        // ignore
                        } catch (Exception e) {
                            exceptions.add(e);
                        }
                    }
                    latch.countDown();
                } catch (Exception e) {
                    exceptions.add(e);
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
                    Thread.sleep(500);
                    rebalancingToken.set(true);
                    checkConsistentMetadata(finalCluster, serverList);
                } catch (Exception e) {
                    exceptions.add(e);
                } finally {
                    // loop.
                    try {
                        latch.await(300, TimeUnit.SECONDS);
                        stopServer(serverList);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        });
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions !!");
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testServerSideRouting ", ae);
        throw ae;
    }
}

Example 25 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testRebalanceCleanSecondary.

@Test(timeout = 600000)
public void testRebalanceCleanSecondary() throws Exception {
    logger.info("Starting testRebalanceCleanSecondary");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 3 }, { 1 }, { 2 } });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Lists.newArrayList(3));
        // start servers 0 , 1, 2
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("enable.repair", "true");
        List<Integer> serverList = Arrays.asList(0, 1, 2);
        currentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
        try {
            AdminClient adminClient = rebalanceKit.controller.getAdminClient();
            populateData(currentCluster, rwStoreDefWithReplication, adminClient, false);
            // Figure out the positive and negative keys to check
            List<ByteArray> positiveTestKeyList = sampleKeysFromPartition(adminClient, 0, rwStoreDefWithReplication.getName(), Arrays.asList(3), 20);
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2));
            checkConsistentMetadata(finalCluster, serverList);
            // Do the cleanup operation
            for (int i = 0; i < 3; i++) {
                adminClient.storeMntOps.repairJob(i);
            }
            // wait for the repairs to complete
            for (int i = 0; i < 3; i++) {
                ServerTestUtils.waitForAsyncOperationOnServer(serverMap.get(i), "Repair", 5000);
            }
            // do the positive tests
            checkForKeyExistence(adminClient, 0, rwStoreDefWithReplication.getName(), positiveTestKeyList);
            logger.info("[Secondary] Successful clean after Rebalancing");
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testRWRebalanceCleanSecondary ", ae);
        throw ae;
    }
}