use of voldemort.utils.Pair in project voldemort by voldemort.
the class CatBdbStore method main.
public static void main(String[] args) throws Exception {
if (args.length != 2)
Utils.croak("USAGE: java " + CatBdbStore.class.getName() + " bdb_dir" + " storeName" + " server.properties.path");
String bdbDir = args[0];
String storeName = args[1];
String serverProperties = args[2];
VoldemortConfig config = new VoldemortConfig(new Props(new File(serverProperties)));
EnvironmentConfig environmentConfig = new EnvironmentConfig();
environmentConfig.setDurability(Durability.COMMIT_NO_SYNC);
environmentConfig.setAllowCreate(true);
environmentConfig.setTransactional(config.isBdbWriteTransactionsEnabled());
Environment environment = new Environment(new File(bdbDir), environmentConfig);
DatabaseConfig databaseConfig = new DatabaseConfig();
databaseConfig.setAllowCreate(true);
databaseConfig.setTransactional(config.isBdbWriteTransactionsEnabled());
databaseConfig.setSortedDuplicates(false);
Database database = environment.openDatabase(null, storeName, databaseConfig);
StorageEngine<ByteArray, byte[], byte[]> store = null;
if (config.getBdbPrefixKeysWithPartitionId()) {
store = new PartitionPrefixedBdbStorageEngine(storeName, environment, database, new BdbRuntimeConfig(), TestUtils.makeSingleNodeRoutingStrategy());
} else {
store = new BdbStorageEngine(storeName, environment, database, new BdbRuntimeConfig());
}
StorageEngine<String, String, String> stringStore = SerializingStorageEngine.wrap(store, new StringSerializer(), new StringSerializer(), new StringSerializer());
Iterator<Pair<String, Versioned<String>>> iter = stringStore.entries();
while (iter.hasNext()) {
Pair<String, Versioned<String>> entry = iter.next();
System.out.println(entry.getFirst() + " => " + entry.getSecond().getValue());
}
}
use of voldemort.utils.Pair in project voldemort by voldemort.
the class StorageService method calculateStats.
private DataSetStats calculateStats(StorageEngine<ByteArray, byte[], byte[]> store) {
DataSetStats stats = new DataSetStats();
ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = store.entries();
try {
int count = 0;
while (iter.hasNext()) {
Pair<ByteArray, Versioned<byte[]>> pair = iter.next();
VectorClock clock = (VectorClock) pair.getSecond().getVersion();
stats.countEntry(pair.getFirst().length(), pair.getSecond().getValue().length + clock.sizeInBytes());
if (count % 10000 == 0)
logger.debug("Processing key " + count);
count++;
}
} finally {
iter.close();
}
return stats;
}
use of voldemort.utils.Pair in project voldemort by voldemort.
the class BlockingSlopPusherJob method run.
/**
* Loop over entries in the slop table and attempt to push them to the
* deserving server
*/
public void run() {
// don't try to run slop pusher job when rebalancing
if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
return;
}
logger.info("Started blocking slop pusher job at " + new Date());
Cluster cluster = metadataStore.getCluster();
failureDetector.getConfig().setCluster(cluster);
Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
long slopsPushed = 0L;
long attemptedPushes = 0L;
for (Node node : cluster.getNodes()) {
attemptedByNode.put(node.getId(), 0L);
succeededByNode.put(node.getId(), 0L);
}
acquireRepairPermit();
try {
SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
iterator = slopStore.entries();
while (iterator.hasNext()) {
if (Thread.interrupted())
throw new InterruptedException("Slop pusher job cancelled");
try {
Pair<ByteArray, Versioned<Slop>> keyAndVal;
try {
keyAndVal = iterator.next();
} catch (Exception e) {
logger.error("Exception in iterator, escaping the loop ", e);
break;
}
Versioned<Slop> versioned = keyAndVal.getSecond();
Slop slop = versioned.getValue();
int nodeId = slop.getNodeId();
// check for dead slops
if (isSlopDead(cluster, storeNames, versioned.getValue())) {
handleDeadSlop(slopStorageEngine, keyAndVal);
// the next slop.
continue;
}
Node node = cluster.getNodeById(nodeId);
attemptedPushes++;
if (attemptedPushes % 10000 == 0) {
logger.info("Attempted pushing " + attemptedPushes + " slops");
}
Long attempted = attemptedByNode.get(nodeId);
attemptedByNode.put(nodeId, attempted + 1L);
if (failureDetector.isAvailable(node)) {
Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
Long startNs = System.nanoTime();
int nBytes = 0;
try {
nBytes = slop.getKey().length();
if (slop.getOperation() == Operation.PUT) {
store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
} else if (slop.getOperation() == Operation.DELETE) {
nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
store.delete(slop.getKey(), versioned.getVersion());
} else {
logger.error("Unknown slop operation: " + slop.getOperation());
continue;
}
failureDetector.recordSuccess(node, deltaMs(startNs));
slopStore.delete(slop.makeKey(), versioned.getVersion());
slopsPushed++;
// Increment succeeded
Long succeeded = succeededByNode.get(nodeId);
succeededByNode.put(nodeId, succeeded + 1L);
// Throttle the bytes...
throttler.maybeThrottle(nBytes);
} catch (ObsoleteVersionException e) {
// okay it is old, just delete it
slopStore.delete(slop.makeKey(), versioned.getVersion());
slopsPushed++;
// Increment succeeded
Long succeeded = succeededByNode.get(nodeId);
succeededByNode.put(nodeId, succeeded + 1L);
// Throttle the bytes...
throttler.maybeThrottle(nBytes);
} catch (UnreachableStoreException e) {
failureDetector.recordException(node, deltaMs(startNs), e);
}
}
} catch (Exception e) {
logger.error(e, e);
}
}
// Only if we reached here do we update stats
logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
for (int nodeId : succeededByNode.keySet()) {
outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
}
slopStorageEngine.resetStats(outstanding);
} catch (Exception e) {
logger.error(e, e);
} finally {
try {
if (iterator != null)
iterator.close();
} catch (Exception e) {
logger.error("Failed to close iterator.", e);
}
this.repairPermits.release(this.getClass().getCanonicalName());
}
}
use of voldemort.utils.Pair in project voldemort by voldemort.
the class AbstractRebalanceTest method checkEntriesPostRebalance.
/**
* Makes sure that all expected partition-stores are on each server after
* the rebalance.
*
* @param currentCluster
* @param finalCluster
* @param storeDefs
* @param nodeCheckList
* @param baselineTuples
* @param baselineVersions
*/
protected void checkEntriesPostRebalance(Cluster currentCluster, Cluster finalCluster, List<StoreDefinition> storeDefs, List<Integer> nodeCheckList, HashMap<String, String> baselineTuples, HashMap<String, VectorClock> baselineVersions) {
for (StoreDefinition storeDef : storeDefs) {
Map<Integer, Set<Pair<Integer, Integer>>> currentNodeToPartitionTuples = ROTestUtils.getNodeIdToAllPartitions(currentCluster, storeDef, true);
Map<Integer, Set<Pair<Integer, Integer>>> finalNodeToPartitionTuples = ROTestUtils.getNodeIdToAllPartitions(finalCluster, storeDef, true);
for (int nodeId : nodeCheckList) {
Set<Pair<Integer, Integer>> currentPartitionTuples = currentNodeToPartitionTuples.get(nodeId);
Set<Pair<Integer, Integer>> finalPartitionTuples = finalNodeToPartitionTuples.get(nodeId);
HashMap<Integer, List<Integer>> flattenedPresentTuples = ROTestUtils.flattenPartitionTuples(Utils.getAddedInTarget(currentPartitionTuples, finalPartitionTuples));
Store<ByteArray, byte[], byte[]> store = getSocketStore(storeDef.getName(), finalCluster.getNodeById(nodeId).getHost(), finalCluster.getNodeById(nodeId).getSocketPort());
checkGetEntries(finalCluster.getNodeById(nodeId), finalCluster, storeDef, store, flattenedPresentTuples, baselineTuples, baselineVersions);
}
}
}
use of voldemort.utils.Pair in project voldemort by voldemort.
the class ZoneShrinkageEndToEndTest method testAllServersSendingOutSlopsCorrectly.
@Test(timeout = 60000)
public void testAllServersSendingOutSlopsCorrectly() throws InterruptedException {
final Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
final Serializer<Slop> slopValueSerializer = new SlopSerializer();
final SlopSerializer slopSerializer = new SlopSerializer();
StoreDefinition storeDef = storeDefs.get(0);
TestSocketStoreFactory ssf = new TestSocketStoreFactory();
Map<Integer, SocketStore> slopStoresCreatedBeforeShrink = new HashMap<Integer, SocketStore>();
Map<Integer, SocketStore> slopStoresCreatedAfterShrink = new HashMap<Integer, SocketStore>();
// generate for keys each all servers that will be hosted on each server
// except itself (2*N*(N-1) keys)
// Map<Integer slopFinalDestinationNodeId, List<Pair<ByteArray key,
// Integer hostNodeId>>>
Map<Integer, List<Pair<ByteArray, Integer>>> serverKeys = new HashMap<Integer, List<Pair<ByteArray, Integer>>>();
for (Node slopFinalDestinationNode : cluster.getNodes()) {
serverKeys.put(slopFinalDestinationNode.getId(), new ArrayList<Pair<ByteArray, Integer>>());
}
// make socket stores to all servers before shrink
for (Integer nodeId : vservers.keySet()) {
SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
slopStoresCreatedBeforeShrink.put(nodeId, slopStore);
}
for (int i = 0; i < 2; i++) {
for (Integer slopHostId : vservers.keySet()) {
SocketStore slopStore = slopStoresCreatedBeforeShrink.get(slopHostId);
for (Integer destinationNodeId : vservers.keySet()) {
if (!destinationNodeId.equals(slopHostId)) {
ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
}
}
}
}
// update metadata
executeShrinkZone();
logger.info("-------------------------------");
logger.info(" CONNECTING SLOP STORES ");
logger.info("-------------------------------");
// make socket stores to all servers after shrink
for (Integer nodeId : vservers.keySet()) {
SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
slopStoresCreatedAfterShrink.put(nodeId, slopStore);
}
logger.info("-------------------------------");
logger.info(" CONNECTED SLOP STORES ");
logger.info("-------------------------------");
logger.info("-------------------------------");
logger.info(" SENDING SLOPS ");
logger.info("-------------------------------");
for (int i = 0; i < 2; i++) {
for (Integer slopHostId : vservers.keySet()) {
SocketStore slopStore = slopStoresCreatedAfterShrink.get(slopHostId);
for (Integer destinationNodeId : vservers.keySet()) {
if (!destinationNodeId.equals(slopHostId)) {
ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
}
}
}
}
logger.info("-------------------------------");
logger.info(" SENT SLOPS ");
logger.info("-------------------------------");
ServerTestUtils.waitForSlopDrain(vservers, 30000L);
// verify all proper slops is processed properly (arrived or dropped)
boolean hasError = false;
int goodCount = 0;
int errorCount = 0;
for (Integer nodeId : serverKeys.keySet()) {
VoldemortServer vs = vservers.get(nodeId);
Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getStorageEngine(storeDef.getName());
List<Pair<ByteArray, Integer>> keySet = serverKeys.get(nodeId);
for (Pair<ByteArray, Integer> keyHostIdPair : keySet) {
ByteArray key = keyHostIdPair.getFirst();
Integer hostId = keyHostIdPair.getSecond();
Integer nodeZoneId = cluster.getNodeById(nodeId).getZoneId();
List<Versioned<byte[]>> result = store.get(key, null);
if (cluster.getNodeById(nodeId).getZoneId() == droppingZoneId) {
if (!result.isEmpty()) {
logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should be gone but exists\n", key.toString(), nodeId, nodeZoneId, hostId));
hasError = true;
errorCount++;
} else {
goodCount++;
}
} else {
if (result.isEmpty()) {
logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should exist but not\n", key.toString(), nodeId, nodeZoneId, hostId));
hasError = true;
errorCount++;
} else {
goodCount++;
}
}
}
}
logger.info(String.format("Good keys count: %d; Error keys count: %d", goodCount, errorCount));
Assert.assertFalse("Error Occurred BAD:" + errorCount + "; GOOD: " + goodCount + ". Check log.", hasError);
}
Aggregations