Examples with Cluster voldemort.cluster.Cluster used on opensource projects

Example 31 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testShuffleZZAndShuffleAgain.

@Test(timeout = 600000)
public void testShuffleZZAndShuffleAgain() throws Exception {
    logger.info("Starting testShuffleZZAndShuffleAgain");
    // Hacky work around of TOCTOU bind Exception issues. Each test that
    // invokes this method brings servers up & down on the same ports. The
    // OS seems to need a rest between subsequent tests...
    Thread.sleep(TimeUnit.SECONDS.toMillis(5));
    Cluster interimCluster = RebalanceUtils.getInterimCluster(zzCurrent, zzShuffle);
    // start all the servers
    List<Integer> serverList = new ArrayList<Integer>(interimCluster.getNodeIds());
    Map<String, String> configProps = new HashMap<String, String>();
    configProps.put("admin.max.threads", "5");
    interimCluster = startServers(interimCluster, zzStoresXml, serverList, configProps);
    // Populate cluster with data
    for (StoreDefinition storeDef : zzStores) {
        populateData(zzCurrent, storeDef);
    }
    String bootstrapUrl = getBootstrapUrl(interimCluster, 0);
    // Shuffle cluster
    ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, zzShuffle, zzStores);
    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);
    checkConsistentMetadata(zzShuffle, serverList);
    // Now, go from shuffled state, back to the original to ocnfirm
    // subsequent rebalances can be invoked.
    rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, zzCurrent, zzStores);
    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);
    checkConsistentMetadata(zzCurrent, serverList);
    // Done.
    stopServer(serverList);
}

Example 32 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testZonedRebalance.

// TODO: The tests based on this method are susceptible to TOCTOU
// BindException issue since findFreePorts is used to determine the ports
// for localhost:PORT of each node.
/**
     * Scripts the execution of a specific type of zoned rebalance test: sets up
     * cluster based on cCluster plus any new nodes/zones in fCluster,
     * rebalances to fCluster, verifies rebalance was correct.
     * 
     * @param testTag For pretty printing
     * @param cCluster current cluster
     * @param fCluster final cluster
     * @param cStoresXml XML file with current stores xml
     * @param fStoresXml Unused parameter. Included for symmetry in method
     *        declaration.
     * @param cStoreDefs store defs for current cluster (from on cStoresXml)
     * @param fStoreDefs store defs for final cluster.
     * @throws Exception
     */
public void testZonedRebalance(String testTag, Cluster cCluster, Cluster fCluster, String cStoresXml, String fStoresXml, List<StoreDefinition> cStoreDefs, List<StoreDefinition> fStoreDefs) throws Exception {
    logger.info("Starting " + testTag);
    // Hacky work around of TOCTOU bind Exception issues. Each test that
    // invokes this method brings servers up & down on the same ports. The
    // OS seems to need a rest between subsequent tests...
    Thread.sleep(TimeUnit.SECONDS.toMillis(5));
    try {
        Cluster interimCluster = RebalanceUtils.getInterimCluster(cCluster, fCluster);
        // start all the servers
        List<Integer> serverList = new ArrayList<Integer>(interimCluster.getNodeIds());
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        interimCluster = startServers(interimCluster, cStoresXml, serverList, configProps);
        String bootstrapUrl = getBootstrapUrl(interimCluster, 0);
        ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, fCluster, fStoreDefs);
        try {
            for (StoreDefinition storeDef : cStoreDefs) {
                populateData(cCluster, storeDef);
            }
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);
            checkConsistentMetadata(fCluster, serverList);
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in " + testTag + " : ", ae);
        throw ae;
    }
}

Example 33 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testRWRebalance.

@Test(timeout = 600000)
public void testRWRebalance() throws Exception {
    logger.info("Starting testRWRebalance");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(4, 2, new int[] { 0, 0, 1, 1 }, new int[][] { { 0, 2, 4, 6 }, {}, { 1, 3, 5, 7 }, {} });
        Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(2, 6));
        finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 1, Lists.newArrayList(3, 7));
        // start all the servers
        List<Integer> serverList = Arrays.asList(0, 1, 2, 3);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        currentCluster = startServers(currentCluster, storeDefFileWithoutReplication, serverList, configProps);
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithoutReplication);
            rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(1, 2));
            checkConsistentMetadata(finalCluster, serverList);
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testRWRebalance ", ae);
        throw ae;
    }
}

Example 34 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method testProxyGetDuringRebalancing.

@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
    logger.info("Starting testProxyGetDuringRebalancing");
    try {
        Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(4, 2, new int[] { 0, 0, 1, 1 }, new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } });
        Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(2));
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 1, Lists.newArrayList(3));
        final List<Integer> serverList = Arrays.asList(0, 1, 2, 3);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "5");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        try {
            populateData(currentCluster, rwStoreDefWithReplication);
            final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
            final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
            final CountDownLatch latch = new CountDownLatch(2);
            // start get operation.
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        List<String> keys = new ArrayList<String>(testEntries.keySet());
                        while (!rebalancingComplete.get()) {
                            // should always able to get values.
                            int index = (int) (Math.random() * keys.size());
                            // should get a valid value
                            try {
                                Versioned<String> value = storeClientRW.get(keys.get(index));
                                assertNotSame("StoreClient get() should not return null.", null, value);
                                assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
                            } catch (Exception e) {
                                logger.error("Exception in proxy get thread", e);
                                e.printStackTrace();
                                exceptions.add(e);
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Exception in proxy get thread", e);
                        exceptions.add(e);
                    } finally {
                        factory.close();
                        latch.countDown();
                    }
                }
            });
            executors.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        Thread.sleep(500);
                        rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2, 3));
                        Thread.sleep(500);
                        rebalancingComplete.set(true);
                        checkConsistentMetadata(finalCluster, serverList);
                    } catch (Exception e) {
                        exceptions.add(e);
                    } finally {
                        // stop servers
                        try {
                            stopServer(serverList);
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                        latch.countDown();
                    }
                }
            });
            latch.await();
            executors.shutdown();
            executors.awaitTermination(300, TimeUnit.SECONDS);
            // check No Exception
            if (exceptions.size() > 0) {
                for (Exception e : exceptions) {
                    e.printStackTrace();
                }
                fail("Should not see any exceptions.");
            }
        } finally {
            // stop servers
            stopServer(serverList);
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
        throw ae;
    }
}

Example 35 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class ReplaceNodeTest method testNodeDownReplacement.

@Test
public void testNodeDownReplacement() throws Exception {
    final int REPLACEMENT_NODE = 0;
    // This is to simulate the case where a machine failed but hard disk was intact
    // In this case we will move the hard disk to another machine, edit the cluster.xml
    // to point to this host and start this machine.
    // The case is simulated by remembering the cluster of node A.
    // Replace this node with node B. Now create a new server with cluster of
    // node A ( this is to simulate the cluster.xml edit) and data directory of B
    // ( this is to simulate the hard disk move). Now try replacing the node B with
    // newly created node after shutting down the node B.
    Cluster cluster = originalServers[nodeToBeReplaced].getMetadataStore().getCluster();
    List<StoreDefinition> storeDefs = originalServers[nodeToBeReplaced].getMetadataStore().getStoreDefList();
    Node node = originalServers[nodeToBeReplaced].getIdentityNode();
    //Verify the node down scenario first
    final boolean DO_RESTORE = false;
    final boolean STOP_OLD_NODE = true;
    verifyNodeReplacement(nodeToBeReplaced, otherServers, REPLACEMENT_NODE, STOP_OLD_NODE, DO_RESTORE);
    // Now the replacement node is part of the original cluster.
    // Stop the replacement node, assume you moved the hard disk
    // to a new node ( This is done by starting another node)
    // that points to the data directory of the replacement node.
    String baseDirPath = otherServers[REPLACEMENT_NODE].getVoldemortConfig().getVoldemortHome();
    // Using ServerTestUtils stopVoldemortServer also deletes the data
    // directory so using the stop, to simulate the hard disk move.
    otherServers[REPLACEMENT_NODE].stop();
    VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, node.getId(), baseDirPath, cluster, storeDefs, new Properties());
    Assert.assertTrue(config.isSlopEnabled());
    Assert.assertTrue(config.isSlopPusherJobEnabled());
    Assert.assertTrue(config.getAutoPurgeDeadSlops());
    config.setSlopFrequencyMs(8000L);
    config.setEnableNodeIdDetection(this.autoDetectNodeId);
    if (this.autoDetectNodeId) {
        config.setNodeIdImplementation(new NodeIdHostMatcher(nodeToBeReplaced));
    }
    // Though baseDir is used as part of Config, TestUtils, always appends
    // it with node- (nodeId). So forcefully overwriting it here to point to
    // the old directory.
    config.setVoldemortHome(baseDirPath);
    SocketStoreFactory ssf = new TestSocketStoreFactory();
    VoldemortServer hardDiskMovedServer = ServerTestUtils.startVoldemortServer(ssf, config, cluster);
    ssf.close();
    otherServers[REPLACEMENT_NODE] = hardDiskMovedServer;
    final boolean SKIP_RESTORE = true;
    final boolean DONOT_STOP_OLD_NODE = true;
    verifyNodeReplacement(nodeToBeReplaced, otherServers, REPLACEMENT_NODE, DONOT_STOP_OLD_NODE, SKIP_RESTORE);
}