use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.
the class StreamingSlopPusherTest method testServerReplacementWithoutBounce.
@Test
public void testServerReplacementWithoutBounce() throws IOException, InterruptedException {
startServers(0, 2);
// Put into slop store 0
StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
// Generate slops for 1
final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
// Generate slops for 2
final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
pusher.run();
// Give some time for the slops to go over
Thread.sleep(10000);
// Now check if the slops went through and also got deleted
Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
while (entryIterator.hasNext()) {
Versioned<Slop> versionedSlop = entryIterator.next();
Slop nextSlop = versionedSlop.getValue();
StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
} else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
}
// did it get deleted correctly
assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
}
entryIterator = entrySet1.listIterator();
while (entryIterator.hasNext()) {
Versioned<Slop> versionedSlop = entryIterator.next();
Slop nextSlop = versionedSlop.getValue();
// did it get deleted correctly
assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
}
// Check counts
SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
assertEquals(slopEngine.getOutstandingTotal(), 50);
assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
// now replace server 1 with a new host and start it
cluster = ServerTestUtils.updateClusterWithNewHost(cluster, 1);
startServers(1);
// update the meatadata store with the new cluster on node 0 and 2 (the
// two servers that are running)
servers[0].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
servers[2].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
// Give some time for the pusher job to figure out that server1 is up
Thread.sleep(35000);
// start the pusher job again
pusher.run();
// Give some time for the slops to go over
Thread.sleep(10000);
// make sure the slot for server 1 is pushed to the new host
// Now check if the slops went through and also got deleted
entryIterator = entrySet1.listIterator();
while (entryIterator.hasNext()) {
Versioned<Slop> versionedSlop = entryIterator.next();
Slop nextSlop = versionedSlop.getValue();
StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
} else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
}
// did it get deleted correctly
assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
}
}
use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.
the class StreamingSlopPusherTest method testPushForHostSwap.
/**
* Procedure: 1. bring up 2-node cluster; 2. set up slop pusher job with the
* cluster info; 3. replace node 1 by simply changing its port-ids, update
* the cluster metadata to whole cluster; 4. run slop pusher job and it has
* to fail; 5. setCluster in FD and the slop pusher job has to succeed
*
* @throws InterruptedException
* @throws IOException
*/
@Test
public void testPushForHostSwap() throws InterruptedException, IOException {
startServers(0, 1);
// Put into slop store 0
StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
// Generate slops for 1
final List<Versioned<Slop>> entrySet = ServerTestUtils.createRandomSlops(1, 100, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
populateSlops(0, slopStoreNode0, entrySet);
// replace node 1 and update metadata on all nodes
replaceOneNode(1);
Cluster oldCluster = cluster;
// test if node 1 is down
stopServersWithoutRemovingVoldemortHome(1);
// slop should fail here
new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new ThresholdFailureDetector(new FailureDetectorConfig().setCluster(oldCluster).setConnectionVerifier(new AdminSlopStreamingVerifier(oldCluster))), configs[0], new ScanPermitWrapper(1)).run();
Thread.sleep(2000);
assertFalse(verifySlopPushResult(entrySet, slopStoreNode0, 1));
// test if node 1 is up
startServers(1);
// slop push should fail
new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new ThresholdFailureDetector(new FailureDetectorConfig().setCluster(oldCluster).setConnectionVerifier(new AdminSlopStreamingVerifier(oldCluster))), configs[0], new ScanPermitWrapper(1)).run();
Thread.sleep(2000);
Assert.assertTrue(verifySlopPushResult(entrySet, slopStoreNode0, 1));
// Check counts
SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
assertEquals(slopEngine.getOutstandingTotal(), 0);
assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(0));
assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
stopServers(0, 1);
}
use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.
the class SlopPurgeJob method operate.
@Override
public void operate() throws Exception {
logger.info("Purging slops that match any of the following. {Nodes:" + nodesToPurge + "} {Zone:" + zoneToPurge + "} {Stores:" + storesToPurge + "}");
SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
ClosableIterator<Pair<ByteArray, Versioned<Slop>>> slopIterator = slopStore.entries();
Set<Integer> nodesInPurgeZone = null;
if (zoneToPurge != Zone.UNSET_ZONE_ID) {
nodesInPurgeZone = metadataStore.getCluster().getNodeIdsInZone(zoneToPurge);
}
try {
while (slopIterator.hasNext()) {
Pair<ByteArray, Versioned<Slop>> keyAndVal = slopIterator.next();
Versioned<Slop> versioned = keyAndVal.getSecond();
Slop slop = versioned.getValue();
// Determine if the slop qualifies for purging..
boolean purge = false;
if (nodesToPurge.contains(slop.getNodeId())) {
purge = true;
} else if (nodesInPurgeZone != null && nodesInPurgeZone.contains(slop.getNodeId())) {
purge = true;
} else if (storesToPurge.contains(slop.getStoreName())) {
purge = true;
}
// if any one of the filters were met, delete
if (purge) {
numKeysUpdatedThisRun.incrementAndGet();
slopStorageEngine.delete(keyAndVal.getFirst(), versioned.getVersion());
}
numKeysScannedThisRun.incrementAndGet();
throttler.maybeThrottle(1);
if (numKeysScannedThisRun.get() % STAT_RECORDS_INTERVAL == 0) {
logger.info("#Scanned:" + numKeysScannedThisRun + " #PurgedSlops:" + numKeysUpdatedThisRun);
}
}
} catch (Exception e) {
logger.error("Error while purging slops", e);
} finally {
slopIterator.close();
}
logger.info("Completed purging slops. " + "#Scanned:" + numKeysScannedThisRun + " #PurgedSlops:" + numKeysUpdatedThisRun);
}
use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.
the class StreamingSlopPusherJob method run.
public void run() {
// load the metadata before each run, in case the cluster is changed
loadMetadata();
// don't try to run slop pusher job when rebalancing
if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
return;
}
boolean terminatedEarly = false;
Date startTime = new Date();
logger.info("Started streaming slop pusher job at " + startTime);
SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
if (adminClient == null) {
adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(1));
}
if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
// Populating the zone mapping for early termination
zoneMapping.clear();
for (Node n : cluster.getNodes()) {
if (failureDetector.isAvailable(n)) {
Set<Integer> nodes = zoneMapping.get(n.getZoneId());
if (nodes == null) {
nodes = Sets.newHashSet();
zoneMapping.put(n.getZoneId(), nodes);
}
nodes.add(n.getId());
}
}
// Check how many zones are down
int zonesDown = 0;
for (Zone zone : cluster.getZones()) {
if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
zonesDown++;
}
// Terminate early
if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size() && zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
logger.info("Completed streaming slop pusher job at " + startTime + " early because " + zonesDown + " zones are down");
stopAdminClient();
return;
}
}
// Clearing the statistics
AtomicLong attemptedPushes = new AtomicLong(0);
for (Node node : cluster.getNodes()) {
attemptedByNode.put(node.getId(), 0L);
succeededByNode.put(node.getId(), 0L);
}
Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
acquireRepairPermit();
try {
StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
iterator = slopStore.entries();
while (iterator.hasNext()) {
Pair<ByteArray, Versioned<Slop>> keyAndVal;
try {
keyAndVal = iterator.next();
Versioned<Slop> versioned = keyAndVal.getSecond();
// Track the scan progress
if (this.streamStats != null) {
this.streamStats.reportStreamingSlopScan();
}
// Retrieve the node
int nodeId = versioned.getValue().getNodeId();
// check for dead slops
if (isSlopDead(cluster, storeNames, versioned.getValue())) {
handleDeadSlop(slopStorageEngine, keyAndVal);
// ignore it.
continue;
}
Node node = cluster.getNodeById(nodeId);
attemptedPushes.incrementAndGet();
Long attempted = attemptedByNode.get(nodeId);
attemptedByNode.put(nodeId, attempted + 1L);
if (attemptedPushes.get() % 10000 == 0)
logger.info("Attempted pushing " + attemptedPushes + " slops");
if (logger.isTraceEnabled())
logger.trace("Pushing slop for " + versioned.getValue().getNodeId() + " and store " + versioned.getValue().getStoreName() + " of key: " + versioned.getValue().getKey());
if (failureDetector.isAvailable(node)) {
SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
if (slopQueue == null) {
// No previous slop queue, add one
slopQueue = new SynchronousQueue<Versioned<Slop>>();
slopQueues.put(nodeId, slopQueue);
consumerResults.add(consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
}
boolean offered = slopQueue.offer(versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
if (!offered) {
if (logger.isDebugEnabled())
logger.debug("No consumer appeared for slop in " + voldemortConfig.getClientConnectionTimeoutMs() + " ms");
}
readThrottler.maybeThrottle(nBytesRead(keyAndVal));
} else {
logger.trace(node + " declared down, won't push slop");
}
} catch (RejectedExecutionException e) {
throw new VoldemortException("Ran out of threads in executor", e);
}
}
} catch (InterruptedException e) {
logger.warn("Interrupted exception", e);
terminatedEarly = true;
} catch (Exception e) {
logger.error(e, e);
terminatedEarly = true;
} finally {
try {
if (iterator != null)
iterator.close();
} catch (Exception e) {
logger.warn("Failed to close iterator cleanly as database might be closed", e);
}
// Adding the poison pill
for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
try {
slopQueue.put(END);
} catch (InterruptedException e) {
logger.warn("Error putting poison pill", e);
}
}
for (Future result : consumerResults) {
try {
result.get();
} catch (Exception e) {
logger.warn("Exception in consumer", e);
}
}
// Only if exception didn't take place do we update the counts
if (!terminatedEarly) {
Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
for (int nodeId : succeededByNode.keySet()) {
logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
}
slopStorageEngine.resetStats(outstanding);
logger.info("Completed streaming slop pusher job which started at " + startTime);
} else {
for (int nodeId : succeededByNode.keySet()) {
logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
}
logger.info("Completed early streaming slop pusher job which started at " + startTime);
}
// Shut down admin client as not to waste connections
consumerResults.clear();
slopQueues.clear();
stopAdminClient();
this.repairPermits.release(this.getClass().getCanonicalName());
}
}
use of voldemort.store.slop.SlopStorageEngine in project voldemort by voldemort.
the class HintedHandoffFailureTest method customSetup.
/**
* Setup a cluster with 3 nodes, with the following characteristics:
*
* If FAILURE_MODE is FAIL_FIRST_REPLICA_NODE set the first replica store to
* a sleepy force failing store
*
* If FAILURE_MODE is FAIL_ALL_REPLICAS: set all replicas to sleepy force
* failing store
*
* Pseudo master : Standard In-memory store (wrapped by Logging store)
*
* In memory slop stores
*
* @param key The ByteArray representation of the key
* @param failureMode The Failure mode for the replicas
*
* @throws Exception
*/
public List<Integer> customSetup(ByteArray key, FAILURE_MODE failureMode, ReplicaFactor replicaFactor, long sleepTime, long hintDelayTimeMs) throws Exception {
cluster = getThreeNodeCluster();
storeDef = getStoreDef(STORE_NAME, replicaFactor, RoutingStrategyType.CONSISTENT_STRATEGY);
strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryStorageEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
LoggingStore<ByteArray, byte[], byte[]> loggingStore = new LoggingStore<ByteArray, byte[], byte[]>(inMemoryStorageEngine);
VoldemortException e = new UnreachableStoreException("Node down");
ForceFailStore<ByteArray, byte[], byte[]> failureStore = new ForceFailStore<ByteArray, byte[], byte[]>(loggingStore, e);
SleepyStore<ByteArray, byte[], byte[]> sleepyFailureStore = new SleepyStore<ByteArray, byte[], byte[]>(sleepTime, failureStore);
failureStore.setFail(true);
List<Integer> failingNodeIdList = Lists.newArrayList();
List<Node> replicaList = strategy.routeRequest(key.get());
switch(failureMode) {
case FAIL_FIRST_REPLICA_NODE:
failingNodeIdList.add(replicaList.get(1).getId());
break;
case FAIL_ALL_REPLICAS:
for (int nodeId = 1; nodeId < replicaList.size(); nodeId++) {
failingNodeIdList.add(nodeId);
}
break;
}
subStores.clear();
for (int i = 0; i < NUM_NODES_TOTAL; i++) {
if (failingNodeIdList.contains(i)) {
subStores.put(i, sleepyFailureStore);
} else {
subStores.put(i, loggingStore);
}
}
setFailureDetector(subStores);
routedStoreThreadPool = Executors.newFixedThreadPool(NUM_THREADS);
routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
for (Node node : cluster.getNodes()) {
int nodeId = node.getId();
SlopStorageEngine slopStorageEngine = new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(SLOP_STORE_NAME), cluster);
StorageEngine<ByteArray, Slop, byte[]> storageEngine = slopStorageEngine.asSlopStore();
nonblockingSlopStores.put(nodeId, routedStoreFactory.toNonblockingStore(slopStorageEngine));
slopStores.put(nodeId, storageEngine);
}
Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : subStores.entrySet()) nonblockingStores.put(entry.getKey(), routedStoreFactory.toNonblockingStore(entry.getValue()));
store = new DelayedPutPipelineRoutedStore(subStores, nonblockingStores, slopStores, nonblockingSlopStores, cluster, storeDef, failureDetector, hintDelayTimeMs);
return failingNodeIdList;
}
Aggregations