use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.
the class ConsistencyFixWorkerTest method testRepair.
public void testRepair(int[] putNodes, boolean orphan) {
byte[] bKey = TestUtils.randomBytes(10);
String hexKey = ByteUtils.toHexString(bKey);
ByteArray baKey = new ByteArray(bKey);
BadKey badKey;
QueryKeyResult queryKeyResult;
if (!orphan) {
badKey = new BadKey(hexKey, hexKey + "\n");
queryKeyResult = null;
} else {
StringBuilder orphanInput = new StringBuilder();
orphanInput.append(hexKey + "," + "1\n");
List<Versioned<byte[]>> values = new ArrayList<Versioned<byte[]>>(0);
int arbitraryNodeId = 2;
Versioned<byte[]> versioned = TestUtils.getVersioned(TestUtils.randomBytes(25), arbitraryNodeId);
orphanInput.append(ByteUtils.toHexString(versioned.getValue()));
orphanInput.append("," + versioned.toString() + "\n");
values.add(versioned);
badKey = new BadKey(hexKey, orphanInput.toString());
queryKeyResult = new QueryKeyResult(baKey, values);
}
Versioned<byte[]> value = TestUtils.getVersioned(TestUtils.randomBytes(25), 0);
String url = ConsistencyFixTest.setUpCluster();
ConsistencyFix consistencyFix = new ConsistencyFix(url, ConsistencyFixTest.STORE_NAME, 100, 100, false, false);
AdminClient adminClient = consistencyFix.getAdminClient();
System.out.println("Initial get");
for (int i = 0; i < 4; ++i) {
List<Versioned<byte[]>> results;
results = adminClient.storeOps.getNodeKey(ConsistencyFixTest.STORE_NAME, i, baKey);
assertTrue(results.size() == 0);
}
System.out.println("Puts");
for (int putNode : putNodes) {
NodeValue<ByteArray, byte[]> nodeKeyValue;
nodeKeyValue = new NodeValue<ByteArray, byte[]>(putNode, baKey, value);
adminClient.storeOps.putNodeKeyValue(ConsistencyFixTest.STORE_NAME, nodeKeyValue);
}
// Construct normal consistency fix worker
ConsistencyFixWorker consistencyFixWorker = null;
if (!orphan) {
consistencyFixWorker = new ConsistencyFixWorker(badKey, consistencyFix, null);
} else {
consistencyFixWorker = new ConsistencyFixWorker(badKey, consistencyFix, null, queryKeyResult);
}
consistencyFixWorker.run();
System.out.println("Second get");
int expectedNumVersions = 0;
if (putNodes.length > 0) {
expectedNumVersions++;
}
if (orphan) {
expectedNumVersions++;
}
for (int i = 0; i < 4; ++i) {
System.out.println("Node : " + i);
List<Versioned<byte[]>> results;
results = adminClient.storeOps.getNodeKey(ConsistencyFixTest.STORE_NAME, i, baKey);
for (Versioned<byte[]> v : results) {
System.out.println("\t" + v.getVersion());
}
assertTrue(results.size() == expectedNumVersions);
}
}
use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.
the class StreamingSlopPusherJob method run.
public void run() {
// load the metadata before each run, in case the cluster is changed
loadMetadata();
// don't try to run slop pusher job when rebalancing
if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
return;
}
boolean terminatedEarly = false;
Date startTime = new Date();
logger.info("Started streaming slop pusher job at " + startTime);
SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
if (adminClient == null) {
adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(1));
}
if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
// Populating the zone mapping for early termination
zoneMapping.clear();
for (Node n : cluster.getNodes()) {
if (failureDetector.isAvailable(n)) {
Set<Integer> nodes = zoneMapping.get(n.getZoneId());
if (nodes == null) {
nodes = Sets.newHashSet();
zoneMapping.put(n.getZoneId(), nodes);
}
nodes.add(n.getId());
}
}
// Check how many zones are down
int zonesDown = 0;
for (Zone zone : cluster.getZones()) {
if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
zonesDown++;
}
// Terminate early
if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size() && zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
logger.info("Completed streaming slop pusher job at " + startTime + " early because " + zonesDown + " zones are down");
stopAdminClient();
return;
}
}
// Clearing the statistics
AtomicLong attemptedPushes = new AtomicLong(0);
for (Node node : cluster.getNodes()) {
attemptedByNode.put(node.getId(), 0L);
succeededByNode.put(node.getId(), 0L);
}
Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
acquireRepairPermit();
try {
StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
iterator = slopStore.entries();
while (iterator.hasNext()) {
Pair<ByteArray, Versioned<Slop>> keyAndVal;
try {
keyAndVal = iterator.next();
Versioned<Slop> versioned = keyAndVal.getSecond();
// Track the scan progress
if (this.streamStats != null) {
this.streamStats.reportStreamingSlopScan();
}
// Retrieve the node
int nodeId = versioned.getValue().getNodeId();
// check for dead slops
if (isSlopDead(cluster, storeNames, versioned.getValue())) {
handleDeadSlop(slopStorageEngine, keyAndVal);
// ignore it.
continue;
}
Node node = cluster.getNodeById(nodeId);
attemptedPushes.incrementAndGet();
Long attempted = attemptedByNode.get(nodeId);
attemptedByNode.put(nodeId, attempted + 1L);
if (attemptedPushes.get() % 10000 == 0)
logger.info("Attempted pushing " + attemptedPushes + " slops");
if (logger.isTraceEnabled())
logger.trace("Pushing slop for " + versioned.getValue().getNodeId() + " and store " + versioned.getValue().getStoreName() + " of key: " + versioned.getValue().getKey());
if (failureDetector.isAvailable(node)) {
SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
if (slopQueue == null) {
// No previous slop queue, add one
slopQueue = new SynchronousQueue<Versioned<Slop>>();
slopQueues.put(nodeId, slopQueue);
consumerResults.add(consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
}
boolean offered = slopQueue.offer(versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
if (!offered) {
if (logger.isDebugEnabled())
logger.debug("No consumer appeared for slop in " + voldemortConfig.getClientConnectionTimeoutMs() + " ms");
}
readThrottler.maybeThrottle(nBytesRead(keyAndVal));
} else {
logger.trace(node + " declared down, won't push slop");
}
} catch (RejectedExecutionException e) {
throw new VoldemortException("Ran out of threads in executor", e);
}
}
} catch (InterruptedException e) {
logger.warn("Interrupted exception", e);
terminatedEarly = true;
} catch (Exception e) {
logger.error(e, e);
terminatedEarly = true;
} finally {
try {
if (iterator != null)
iterator.close();
} catch (Exception e) {
logger.warn("Failed to close iterator cleanly as database might be closed", e);
}
// Adding the poison pill
for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
try {
slopQueue.put(END);
} catch (InterruptedException e) {
logger.warn("Error putting poison pill", e);
}
}
for (Future result : consumerResults) {
try {
result.get();
} catch (Exception e) {
logger.warn("Exception in consumer", e);
}
}
// Only if exception didn't take place do we update the counts
if (!terminatedEarly) {
Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
for (int nodeId : succeededByNode.keySet()) {
logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
}
slopStorageEngine.resetStats(outstanding);
logger.info("Completed streaming slop pusher job which started at " + startTime);
} else {
for (int nodeId : succeededByNode.keySet()) {
logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
}
logger.info("Completed early streaming slop pusher job which started at " + startTime);
}
// Shut down admin client as not to waste connections
consumerResults.clear();
slopQueues.clear();
stopAdminClient();
this.repairPermits.release(this.getClass().getCanonicalName());
}
}
use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.
the class ZoneShrinkageEndToEndTest method executeShrinkZone.
public void executeShrinkZone() {
AdminClient adminClient;
logger.info("-------------------------------");
logger.info(" UPDATING BOTH XML ");
logger.info("-------------------------------");
adminClient = new AdminClient(bootstrapURL);
// set stores metadata (simulating admin tools)
String validatedStoresXML = storeDefinitionsMapper.writeStoreList(storeDefinitionsMapper.readStoreList(new StringReader(finalStoresXML)));
String validatedClusterXML = clusterMapper.writeCluster(clusterMapper.readCluster(new StringReader(finalClusterXML)));
VoldemortAdminTool.executeSetMetadataPair(-1, adminClient, "cluster.xml", validatedClusterXML, "stores.xml", validatedStoresXML);
adminClient.close();
logger.info("-------------------------------");
logger.info(" UPDATED BOTH XML ");
logger.info("-------------------------------");
}
use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.
the class ZonedRebalanceNonContiguousZonesTest method testRebalanceCleanPrimarySecondary.
@Test(timeout = 600000)
public void testRebalanceCleanPrimarySecondary() throws Exception {
logger.info("Starting testRebalanceCleanPrimary");
try {
int[] zoneIds = new int[] { 1, 3 };
int[][] nodesPerZone = new int[][] { { 3, 4, 5 }, { 9, 10, 11 } };
int[][] partitionMap = new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } };
Cluster currentCluster = ServerTestUtils.getLocalNonContiguousZonedCluster(zoneIds, nodesPerZone, partitionMap, ClusterTestUtils.getClusterPorts());
Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 5, Lists.newArrayList(7));
finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 11, Lists.newArrayList(6));
/**
* original server partition ownership
*
* [s3 : p0,p3,p4,p5,p6,p7] [s4 : p1-p7] [s5 : p1,p2] [s9 : p0,p1,p2,p3,p6,p7] [s10 : p1-p7] [s11 : p4,p5]
*
* final server partition ownership
*
* [s3 : p0,p2,p3,p4,p5,p6,p7] [s4 : p0,p1] [s5 : p1-p7] [s9 : p0.p1,p2,p3,p5,p6,p7]
* [s10 : p0,p1,p2,p3,p4,p7] [s11 : p4,p5,p6]
*/
// start servers
List<Integer> serverList = Arrays.asList(3, 4, 5, 9, 10, 11);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("enable.repair", "true");
currentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
String bootstrapUrl = getBootstrapUrl(currentCluster, 3);
ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, finalCluster);
try {
populateData(currentCluster, rwStoreDefWithReplication);
AdminClient admin = rebalanceKit.controller.getAdminClient();
List<ByteArray> p6KeySamples = sampleKeysFromPartition(admin, 4, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
List<ByteArray> p1KeySamples = sampleKeysFromPartition(admin, 4, rwStoreDefWithReplication.getName(), Arrays.asList(1), 20);
List<ByteArray> p3KeySamples = sampleKeysFromPartition(admin, 3, rwStoreDefWithReplication.getName(), Arrays.asList(3), 20);
List<ByteArray> p2KeySamples = sampleKeysFromPartition(admin, 4, rwStoreDefWithReplication.getName(), Arrays.asList(2), 20);
List<ByteArray> p7KeySamples = sampleKeysFromPartition(admin, 10, rwStoreDefWithReplication.getName(), Arrays.asList(7), 20);
rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(3, 4, 5, 9));
checkConsistentMetadata(finalCluster, serverList);
// Do the cleanup operation
for (int i = 0; i < 6; i++) {
admin.storeMntOps.repairJob(serverList.get(i));
}
// wait for the repairs to complete
for (int i = 0; i < 6; i++) {
ServerTestUtils.waitForAsyncOperationOnServer(serverMap.get(serverList.get(i)), "Repair", 5000);
}
// confirm a primary movement in zone 1 : P6 : s4 -> S5. The zone 1 primary changes when
// p6 moves cross zone check for existence of p6 in server 5,
checkForKeyExistence(admin, 5, rwStoreDefWithReplication.getName(), p6KeySamples);
// confirm a secondary movement in zone 1.. p2 : s4 -> s3 check
// for its existence in server 3
checkForKeyExistence(admin, 3, rwStoreDefWithReplication.getName(), p2KeySamples);
// check for its absernce in server 4
// also check that p1 is stable in server 4 [primary stability]
checkForKeyExistence(admin, 4, rwStoreDefWithReplication.getName(), p1KeySamples);
// check that p3 is stable in server 3 [Secondary stability]
checkForKeyExistence(admin, 3, rwStoreDefWithReplication.getName(), p3KeySamples);
// finally, test for server 10 which now became the secondary
// for p7 from being a primary before
checkForKeyExistence(admin, 10, rwStoreDefWithReplication.getName(), p7KeySamples);
} finally {
// stop servers
stopServer(serverList);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testRebalanceCleanPrimarySecondary ", ae);
throw ae;
}
}
use of voldemort.client.protocol.admin.AdminClient in project voldemort by voldemort.
the class AbstractZonedRebalanceTest method testProxyPutDuringRebalancing.
@Test(timeout = 600000)
public void testProxyPutDuringRebalancing() throws Exception {
logger.info("Starting testProxyPutDuringRebalancing");
try {
Cluster currentCluster = ServerTestUtils.getLocalZonedCluster(6, 2, new int[] { 0, 0, 0, 1, 1, 1 }, new int[][] { { 0 }, { 1, 6 }, { 2 }, { 3 }, { 4, 7 }, { 5 } });
Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Lists.newArrayList(7));
finalCluster = UpdateClusterUtils.createUpdatedCluster(finalCluster, 5, Lists.newArrayList(6));
/**
* Original partition map
*
* [s0 : p0] [s1 : p1, p6] [s2 : p2]
*
* [s3 : p3] [s4 : p4, p7] [s5 : p5]
*
* final server partition ownership
*
* [s0 : p0] [s1 : p1] [s2 : p2, p7]
*
* [s3 : p3] [s4 : p4] [s5 : p5, p6]
*
* Note that rwStoreDefFileWithReplication is a "2/1/1" store def.
*
* Original server n-ary partition ownership
*
* [s0 : p0, p3-7] [s1 : p0-p7] [s2 : p1-2]
*
* [s3 : p0-3, p6-7] [s4 : p0-p7] [s5 : p4-5]
*
* final server n-ary partition ownership
*
* [s0 : p0, p2-7] [s1 : p0-1] [s2 : p1-p7]
*
* [s3 : p0-3, p5-7] [s4 : p0-4, p7] [s5 : p4-6]
*/
List<Integer> serverList = Arrays.asList(0, 1, 2, 3, 4, 5);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "5");
final Cluster updatedCurrentCluster = startServers(currentCluster, rwStoreDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingComplete = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
// Its is imperative that we test in a single shot since multiple
// batches would mean the proxy bridges being torn down and
// established multiple times and we cannot test against the source
// cluster topology then. getRebalanceKit uses batch size of
// infinite, so this should be fine.
String bootstrapUrl = getBootstrapUrl(updatedCurrentCluster, 0);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
populateData(currentCluster, rwStoreDefWithReplication);
final AdminClient adminClient = rebalanceKit.controller.getAdminClient();
// the plan would cause these partitions to move:
// Partition : Donor -> stealer
//
// p2 (Z-SEC) : s1 -> s0
// p3-6 (Z-PRI) : s1 -> s2
// p7 (Z-PRI) : s0 -> s2
//
// p5 (Z-SEC): s4 -> s3
// p6 (Z-PRI): s4 -> s5
//
// :. rebalancing will run on servers 0, 2, 3, & 5
final List<ByteArray> movingKeysList = sampleKeysFromPartition(adminClient, 1, rwStoreDefWithReplication.getName(), Arrays.asList(6), 20);
assertTrue("Empty list of moving keys...", movingKeysList.size() > 0);
final AtomicBoolean rebalancingStarted = new AtomicBoolean(false);
final AtomicBoolean proxyWritesDone = new AtomicBoolean(false);
final HashMap<String, String> baselineTuples = new HashMap<String, String>(testEntries);
final HashMap<String, VectorClock> baselineVersions = new HashMap<String, VectorClock>();
for (String key : baselineTuples.keySet()) {
baselineVersions.put(key, new VectorClock());
}
final CountDownLatch latch = new CountDownLatch(2);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
SocketStoreClientFactory factory = null;
try {
// wait for the rebalancing to begin
List<VoldemortServer> serverList = Lists.newArrayList(serverMap.get(0), serverMap.get(2), serverMap.get(3), serverMap.get(5));
while (!rebalancingComplete.get()) {
Iterator<VoldemortServer> serverIterator = serverList.iterator();
while (serverIterator.hasNext()) {
VoldemortServer server = serverIterator.next();
if (ByteUtils.getString(server.getMetadataStore().get(MetadataStore.SERVER_STATE_KEY, null).get(0).getValue(), "UTF-8").compareTo(VoldemortState.REBALANCING_MASTER_SERVER.toString()) == 0) {
logger.info("Server " + server.getIdentityNode().getId() + " transitioned into REBALANCING MODE");
serverIterator.remove();
}
}
if (serverList.size() == 0) {
rebalancingStarted.set(true);
break;
}
}
if (rebalancingStarted.get()) {
factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(getBootstrapUrl(updatedCurrentCluster, 0)).setEnableLazy(false).setSocketTimeout(120, TimeUnit.SECONDS).setClientZoneId(1));
final StoreClient<String, String> storeClientRW = new DefaultStoreClient<String, String>(testStoreNameRW, null, factory, 3);
// now with zero vector clock
for (ByteArray movingKey : movingKeysList) {
try {
String keyStr = ByteUtils.getString(movingKey.get(), "UTF-8");
String valStr = "proxy_write";
storeClientRW.put(keyStr, valStr);
baselineTuples.put(keyStr, valStr);
// all these keys will have [5:1] vector
// clock is node 5 is the new pseudo master
baselineVersions.get(keyStr).incrementVersion(5, System.currentTimeMillis());
proxyWritesDone.set(true);
if (rebalancingComplete.get()) {
break;
}
} catch (InvalidMetadataException e) {
// let this go
logger.error("Encountered an invalid metadata exception.. ", e);
}
}
}
} catch (Exception e) {
logger.error("Exception in proxy write thread..", e);
exceptions.add(e);
} finally {
if (factory != null)
factory.close();
latch.countDown();
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
rebalanceKit.rebalance();
} catch (Exception e) {
logger.error("Error in rebalancing... ", e);
exceptions.add(e);
} finally {
rebalancingComplete.set(true);
latch.countDown();
}
}
});
latch.await();
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
assertEquals("Client did not see all server transition into rebalancing state", rebalancingStarted.get(), true);
assertEquals("Not enough time to begin proxy writing", proxyWritesDone.get(), true);
checkEntriesPostRebalance(updatedCurrentCluster, finalCluster, Lists.newArrayList(rwStoreDefWithReplication), Arrays.asList(0, 1, 2, 3, 4, 5), baselineTuples, baselineVersions);
checkConsistentMetadata(finalCluster, serverList);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions.");
}
// check that the proxy writes were made to the original donor, node
// 1
List<ClockEntry> clockEntries = new ArrayList<ClockEntry>(serverList.size());
for (Integer nodeid : serverList) clockEntries.add(new ClockEntry(nodeid.shortValue(), System.currentTimeMillis()));
VectorClock clusterXmlClock = new VectorClock(clockEntries, System.currentTimeMillis());
for (Integer nodeid : serverList) adminClient.metadataMgmtOps.updateRemoteCluster(nodeid, currentCluster, clusterXmlClock);
adminClient.setAdminClientCluster(currentCluster);
checkForTupleEquivalence(adminClient, 1, testStoreNameRW, movingKeysList, baselineTuples, baselineVersions);
// stop servers
try {
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testProxyPutDuringRebalancing ", ae);
throw ae;
}
}
Aggregations