use of voldemort.versioning.Versioned in project voldemort by voldemort.
the class BdbRevertPidScanToBase method transfer.
@Override
public void transfer() throws Exception {
cursor = srcDB.openCursor(null, null);
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry valueEntry = new DatabaseEntry();
VersionedSerializer<byte[]> versionedSerializer = new VersionedSerializer<byte[]>(new IdentitySerializer());
List<Versioned<byte[]>> vals;
long startTime = System.currentTimeMillis();
int scanCount = 0;
int keyCount = 0;
while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
keyCount++;
vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
// pull out the real key
byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
for (Versioned<byte[]> val : vals) {
OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), new DatabaseEntry(versionedSerializer.toBytes(val)));
if (OperationStatus.SUCCESS != putStatus) {
String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(stripedKey);
logger.error(errorStr);
throw new Exception(errorStr);
}
scanCount++;
}
if (scanCount % 1000000 == 0)
logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
use of voldemort.versioning.Versioned in project voldemort by voldemort.
the class BdbRevertPidScanToNewDup method transfer.
@Override
public void transfer() throws Exception {
cursor = srcDB.openCursor(null, null);
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry valueEntry = new DatabaseEntry();
List<Versioned<byte[]>> vals;
long startTime = System.currentTimeMillis();
int scanCount = 0;
int keyCount = 0;
while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
keyCount++;
vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
scanCount += vals.size();
// pull out the real key
byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry);
if (OperationStatus.SUCCESS != putStatus) {
String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
logger.error(errorStr);
throw new Exception(errorStr);
}
if (scanCount % 1000000 == 0)
logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
use of voldemort.versioning.Versioned in project voldemort by voldemort.
the class VersionedPutPruneJob method operate.
@Override
public void operate() throws Exception {
StoreDefinition storeDef = StoreDefinitionUtils.getStoreDefinitionWithName(metadataStore.getStoreDefList(), storeName);
if (storeDef == null) {
throw new VoldemortException("Unknown store " + storeName);
}
if (isWritableStore(storeDef)) {
// Lets generate routing strategy for this storage engine
StoreRoutingPlan routingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
logger.info("Pruning store " + storeDef.getName());
StorageEngine<ByteArray, byte[], byte[]> engine = storeRepo.getStorageEngine(storeDef.getName());
iterator = engine.keys();
long itemsScanned = 0;
long numPrunedKeys = 0;
while (iterator.hasNext()) {
ByteArray key = iterator.next();
KeyLockHandle<byte[]> lockHandle = null;
try {
lockHandle = engine.getAndLock(key);
List<Versioned<byte[]>> vals = lockHandle.getValues();
List<Integer> keyReplicas = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key.get()));
MutableBoolean didPrune = new MutableBoolean(false);
List<Versioned<byte[]>> prunedVals = pruneNonReplicaEntries(vals, keyReplicas, didPrune);
// happened. Optimization to reduce load on storage
if (didPrune.booleanValue()) {
List<Versioned<byte[]>> resolvedVals = VectorClockUtils.resolveVersions(prunedVals);
// TODO this is only implemented for BDB for now
lockHandle.setValues(resolvedVals);
engine.putAndUnlock(key, lockHandle);
numPrunedKeys = this.numKeysUpdatedThisRun.incrementAndGet();
} else {
// if we did not prune, still need to let go of the lock
engine.releaseLock(lockHandle);
}
itemsScanned = this.numKeysScannedThisRun.incrementAndGet();
throttler.maybeThrottle(1);
if (itemsScanned % STAT_RECORDS_INTERVAL == 0) {
logger.info("#Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
}
} catch (Exception e) {
throw e;
} finally {
if (lockHandle != null && !lockHandle.isClosed()) {
engine.releaseLock(lockHandle);
}
}
}
logger.info("Completed store " + storeDef.getName() + " #Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
}
}
use of voldemort.versioning.Versioned in project voldemort by voldemort.
the class AbstractNonZonedRebalanceTest method testServerSideRouting.
@Test(timeout = 600000)
public void testServerSideRouting() throws Exception {
logger.info("Starting testServerSideRouting");
try {
final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
final List<Integer> serverList = Arrays.asList(0, 1);
Map<String, String> configProps = new HashMap<String, String>();
configProps.put("admin.max.threads", "50");
configProps.put("enable.server.routing", "true");
final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
ExecutorService executors = Executors.newFixedThreadPool(2);
final AtomicBoolean rebalancingToken = new AtomicBoolean(false);
final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
int maxParallel = 2;
final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
// Populate the two stores
populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
Node node = updatedCurrentCluster.getNodeById(1);
final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRW = getSocketStore(testStoreNameRW, node.getHost(), node.getSocketPort(), true);
final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRO = getSocketStore(testStoreNameRO, node.getHost(), node.getSocketPort(), true);
final CountDownLatch latch = new CountDownLatch(1);
// start get operation.
executors.execute(new Runnable() {
@Override
public void run() {
try {
List<String> keys = new ArrayList<String>(testEntries.keySet());
while (!rebalancingToken.get()) {
// should always able to get values.
int index = (int) (Math.random() * keys.size());
// should get a valid value
try {
List<Versioned<byte[]>> values = serverSideRoutingStoreRW.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
assertEquals("serverSideRoutingStore should return value.", 1, values.size());
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
values = serverSideRoutingStoreRO.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
assertEquals("serverSideRoutingStore should return value.", 1, values.size());
assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
} catch (UnreachableStoreException e) {
// ignore
} catch (Exception e) {
exceptions.add(e);
}
}
latch.countDown();
} catch (Exception e) {
exceptions.add(e);
}
}
});
executors.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(500);
rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
Thread.sleep(500);
rebalancingToken.set(true);
checkConsistentMetadata(finalCluster, serverList);
} catch (Exception e) {
exceptions.add(e);
} finally {
// loop.
try {
latch.await(300, TimeUnit.SECONDS);
stopServer(serverList);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
});
executors.shutdown();
executors.awaitTermination(300, TimeUnit.SECONDS);
// check No Exception
if (exceptions.size() > 0) {
for (Exception e : exceptions) {
e.printStackTrace();
}
fail("Should not see any exceptions !!");
}
} catch (AssertionError ae) {
logger.error("Assertion broken in testServerSideRouting ", ae);
throw ae;
}
}
use of voldemort.versioning.Versioned in project voldemort by voldemort.
the class AbstractRebalanceTest method checkForKeyExistence.
/**
* REFACTOR: these should belong AdminClient so existence checks can be done
* easily across the board
*
* @param admin
* @param serverId
* @param store
* @param keyList
*/
protected void checkForKeyExistence(AdminClient admin, int serverId, String store, List<ByteArray> keyList) {
// do the positive tests
Iterator<QueryKeyResult> positiveTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
while (positiveTestResultsItr.hasNext()) {
QueryKeyResult item = positiveTestResultsItr.next();
ByteArray key = item.getKey();
List<Versioned<byte[]>> vals = item.getValues();
Exception e = item.getException();
assertEquals("Error fetching key " + key, null, e);
assertEquals("Value not found for key " + key, true, vals != null & vals.size() != 0);
}
}
Aggregations