use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.
the class AbstractZonedRebalanceTest method testProxyGetDuringRebalancing.
@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
logger.info("Starting testProxyGetDuringRebalancing");
try {
Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(4, 2, new int[] { 0, 0, 1, 1 }, new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } });
Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(2));
final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 1, Lists.newArrayList(3));
final List<Integer> serverList = Arrays.asList(0, 1, 2, 3);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
try {
populateData(currentCluster, rwStoreDefWithReplication);
final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
try {
List<String> keys = new ArrayList<String>(testEntries.keySet());
while (!rebalancingComplete.get()) {
// should always able to get values.
int index = (int) (Math.random() * keys.size());
// should get a valid value
try {
Versioned<String> value = storeClientRW.get(keys.get(index));
assertNotSame("StoreClient get() should not return null.", null, value);
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
} catch (Exception e) {
logger.error("Exception in proxy get thread", e);
e.printStackTrace();
exceptions.add(e);
}
}
} catch (Exception e) {
logger.error("Exception in proxy get thread", e);
exceptions.add(e);
} finally {
factory.close();
latch.countDown();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(500);
rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1, 2, 3));
Thread.sleep(500);
rebalancingComplete.set(true);
checkConsistentMetadata(finalCluster, serverList);
} catch (Exception e) {
exceptions.add(e);
} finally {
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
} finally {
// stop servers
stopServer(serverList);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
throw ae;
}
}
use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.
the class ZonedRebalanceNonContiguousZonesTest method testProxyPutDuringRebalancing.
@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
logger.info("Starting testProxyPutDuringRebalancing");
try {
int[] zoneIds = new int[] { 1, 3 };
int[][] nodesPerZone = new int[][] { { 3, 4, 5 }, { 9, 10, 11 } };
int[][] partitionMap = new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } };
Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 5, Lists.newArrayList(7));
finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 11, Lists.newArrayList(6));
/**
* Original partition map
*
* [s3 : p0] [s4 : p1, p6] [s5 : p2]
*
* [s9 : p3] [s10 : p4, p7] [s11 : p5]
*
* final server partition ownership
*
* [s3 : p0] [s4 : p1] [s5 : p2, p7]
*
* [s9 : p3] [s10 : p4] [s11 : p5, p6]
*
* Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
*
* Original server n-ary partition ownership
*
* [s3 : p0, p3-7] [s4 : p0-p7] [s5 : p1-2]
*
* [s9 : p0-3, p6-7] [s10 : p0-p7] [s11 : p4-5]
*
* final server n-ary partition ownership
*
* [s3 : p0, p2-7] [s4 : p0-1] [s5 : p1-p7]
*
* [s9 : p0-3, p5-7] [s10 : p0-4, p7] [s11 : p4-6]
*/
List<Integer> serverList = Arrays.asList(3, 4, 5, 9, 10, 11);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
// Its is imperative that we test in a single shot since multiple batches would mean the proxy bridges
// being torn down and established multiple times and we cannot test against the source
// cluster topology then. getRebalanceKit uses batch size of infinite, so this should be fine.
String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
populateData(currentCluster, rwStoreDefWithReplication);
final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
// the plan would cause these partitions to move:
// Partition : Donor -> stealer
//
// p2 (Z-SEC) : s4 -> s3
// p3-6 (Z-PRI) : s4 -> s5
// p7 (Z-PRI) : s3 -> s5
//
// p5 (Z-SEC): s10 -> s9
// p6 (Z-PRI): s10 -> s11
//
// Rebalancing will run on servers 3, 5, 9, & 11
final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 4, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
for (String key : baselineTuples.keySet()) {
baselineVersions.put(key, new VectorClock());
}
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
SocketStoreClientFactory factory = null;
try {
// wait for the rebalancing to begin
List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(3), serverMap.get(5), serverMap.get(9), serverMap.get(11));
while (!rebalancingComplete.get()) {
Iterator<VoldemortServer> serverIterator = serverList.iterator();
while (serverIterator.hasNext()) {
VoldemortServer server = serverIterator.next();
if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
serverIterator.remove();
}
}
if (serverList.size() == 0) {
rebalancingStarted.set(true);
break;
}
}
if (rebalancingStarted.get()) {
factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(3));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
// Initially, all data now with zero vector clock
for (ByteArray movingKey : movingKeysList) {
try {
String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
String valStr = "proxy_write";
storeClientRW.put(keyStr, valStr);
baselineTuples.put(keyStr, valStr);
baselineVersions.get(keyStr).incrementVersion(11, System.currentTimeMillis());
proxyWritesDone.set(true);
if (rebalancingComplete.get()) {
break;
}
} catch (InvalidMetadataException e) {
logger.error("Encountered an invalid metadata exception.. ", e);
}
}
}
} catch (Exception e) {
logger.error("Exception in proxy write thread..", e);
exceptions.add(e);
} finally {
if (factory != null)
factory.close();
latch.countDown();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
rebalanceKit.rebalance();
} catch (Exception e) {
logger.error("Error in rebalancing... ", e);
exceptions.add(e);
} finally {
rebalancingComplete.set(true);
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(3, 4, 5, 9, 10, 11), baselineTuples, baselineVersions);
checkConsistentMetadata(finalCluster, serverList);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
// check that the proxy writes were made to the original donor, node 4
List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
adminClient.setAdminClientCluster(currentCluster);
checkForTupleEquivalence(adminClient, 4, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
throw ae;
}
}
use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.
the class ZonedRebalanceNonContiguousZonesTest method testProxyGetDuringRebalancing.
@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
logger.info("Starting testProxyGetDuringRebalancing");
try {
int[] zoneIds = new int[] { 1, 3 };
int[][] nodesPerZone = new int[][] { { 3, 4 }, { 9, 10 } };
int[][] partitionMap = new int[][] { { 0, 2, 4 }, { 6 }, { 1, 3, 5 }, { 7 } };
Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
Cluster tmpfinalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 10, Lists.newArrayList(2));
final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(tmpfinalCluster, 4, Lists.newArrayList(3));
// start servers
final List<Integer> serverList = Arrays.asList(3, 4, 9, 10);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 3);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
try {
populateData(currentCluster, rwStoreDefWithReplication);
final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(currentCluster, 3)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(rwStoreDefWithReplication.getName(), null, factory, 3);
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
try {
List<String> keys = new ArrayList<String>(testEntries.keySet());
while (!rebalancingComplete.get()) {
// should always able to get values.
int index = (int) (Math.random() * keys.size());
// should get a valid value
try {
Versioned<String> value = storeClientRW.get(keys.get(index));
assertNotSame("StoreClient get() should not return null.", null, value);
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
} catch (Exception e) {
logger.error("Exception in proxy get thread", e);
e.printStackTrace();
exceptions.add(e);
}
}
} catch (Exception e) {
logger.error("Exception in proxy get thread", e);
exceptions.add(e);
} finally {
factory.close();
latch.countDown();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(500);
rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(3, 4, 9, 10));
Thread.sleep(500);
rebalancingComplete.set(true);
checkConsistentMetadata(finalCluster, serverList);
} catch (Exception e) {
exceptions.add(e);
} finally {
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
} finally {
// stop servers
stopServer(serverList);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
throw ae;
}
}
use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.
the class AbstractNonZonedRebalanceTest method testProxyGetDuringRebalancing.
@Test(timeout = 600000)
public void testProxyGetDuringRebalancing() throws Exception {
logger.info("Starting testProxyGetDuringRebalancing");
try {
final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
// start servers 0 , 1 only
final List<Integer> serverList = Arrays.asList(0, 1);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
// Populate the two stores
populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
final SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
final StoreClient<String, String> storeClientRO = new DefaultStoreClient<String, String>(testStoreNameRO, null, factory, 3);
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
try {
List<String> keys = new ArrayList<String>(testEntries.keySet());
while (!rebalancingComplete.get()) {
// should always able to get values.
int index = (int) (Math.random() * keys.size());
// should get a valid value
try {
Versioned<String> value = storeClientRW.get(keys.get(index));
assertNotSame("StoreClient get() should not return null.", null, value);
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
value = storeClientRO.get(keys.get(index));
assertNotSame("StoreClient get() should not return null.", null, value);
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), value);
} catch (Exception e) {
logger.error("Exception in online thread", e);
exceptions.add(e);
} finally {
latch.countDown();
}
}
} catch (Exception e) {
logger.error("Exception in proxy get thread", e);
exceptions.add(e);
} finally {
factory.close();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(500);
rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
Thread.sleep(500);
rebalancingComplete.set(true);
checkConsistentMetadata(finalCluster, serverList);
} catch (Exception e) {
exceptions.add(e);
logger.error("Exception in rebalancing thread", e);
} finally {
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyGetDuringRebalancing ", ae);
throw ae;
}
}
use of voldemort.client.DefaultStoreClient in project voldemort by voldemort.
the class AbstractZonedRebalanceTest method testProxyPutDuringRebalancing.
@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
logger.info("Starting testProxyPutDuringRebalancing");
try {
Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(6, 2, new int[] { 0, 0, 0, 1, 1, 1 }, new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } });
Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Lists.newArrayList(7));
finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 5, Lists.newArrayList(6));
/**
* Original partition map
*
* [s0 : p0] [s1 : p1, p6] [s2 : p2]
*
* [s3 : p3] [s4 : p4, p7] [s5 : p5]
*
* final server partition ownership
*
* [s0 : p0] [s1 : p1] [s2 : p2, p7]
*
* [s3 : p3] [s4 : p4] [s5 : p5, p6]
*
* Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
*
* Original server n-ary partition ownership
*
* [s0 : p0, p3-7] [s1 : p0-p7] [s2 : p1-2]
*
* [s3 : p0-3, p6-7] [s4 : p0-p7] [s5 : p4-5]
*
* final server n-ary partition ownership
*
* [s0 : p0, p2-7] [s1 : p0-1] [s2 : p1-p7]
*
* [s3 : p0-3, p5-7] [s4 : p0-4, p7] [s5 : p4-6]
*/
List<Integer> serverList = Arrays.asList(0, 1, 2, 3, 4, 5);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
// Its is imperative that we test in a single shot since multiple
// batches would mean the proxy bridges being torn down and
// established multiple times and we cannot test against the source
// cluster topology then. getRebalanceKit uses batch size of
// infinite, so this should be fine.
String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
populateData(currentCluster, rwStoreDefWithReplication);
final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
// the plan would cause these partitions to move:
// Partition : Donor -> stealer
//
// p2 (Z-SEC) : s1 -> s0
// p3-6 (Z-PRI) : s1 -> s2
// p7 (Z-PRI) : s0 -> s2
//
// p5 (Z-SEC): s4 -> s3
// p6 (Z-PRI): s4 -> s5
//
// :. rebalancing will run on servers 0, 2, 3, & 5
final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 1, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
for (String key : baselineTuples.keySet()) {
baselineVersions.put(key, new VectorClock());
}
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
SocketStoreClientFactory factory = null;
try {
// wait for the rebalancing to begin
List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(0), serverMap.get(2), serverMap.get(3), serverMap.get(5));
while (!rebalancingComplete.get()) {
Iterator<VoldemortServer> serverIterator = serverList.iterator();
while (serverIterator.hasNext()) {
VoldemortServer server = serverIterator.next();
if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
serverIterator.remove();
}
}
if (serverList.size() == 0) {
rebalancingStarted.set(true);
break;
}
}
if (rebalancingStarted.get()) {
factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(1));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
// now with zero vector clock
for (ByteArray movingKey : movingKeysList) {
try {
String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
String valStr = "proxy_write";
storeClientRW.put(keyStr, valStr);
baselineTuples.put(keyStr, valStr);
// all these keys will have [5:1] vector
// clock is node 5 is the new pseudo master
baselineVersions.get(keyStr).incrementVersion(5, System.currentTimeMillis());
proxyWritesDone.set(true);
if (rebalancingComplete.get()) {
break;
}
} catch (InvalidMetadataException e) {
// let this go
logger.error("Encountered an invalid metadata exception.. ", e);
}
}
}
} catch (Exception e) {
logger.error("Exception in proxy write thread..", e);
exceptions.add(e);
} finally {
if (factory != null)
factory.close();
latch.countDown();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
rebalanceKit.rebalance();
} catch (Exception e) {
logger.error("Error in rebalancing... ", e);
exceptions.add(e);
} finally {
rebalancingComplete.set(true);
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(0, 1, 2, 3, 4, 5), baselineTuples, baselineVersions);
checkConsistentMetadata(finalCluster, serverList);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
// check that the proxy writes were made to the original donor, node
// 1
List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
adminClient.setAdminClientCluster(currentCluster);
checkForTupleEquivalence(adminClient, 1, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
throw ae;
}
}
Aggregations