use of voldemort.utils.ByteArray in project voldemort by voldemort.
the class AdminServiceBasicTest method testUpdateSlops.
@Test
public void testUpdateSlops() {
final List<Versioned<Slop>> entrySet = ServerTestUtils.createRandomSlops(0, 10000, testStoreName, "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
Iterator<Versioned<Slop>> slopIterator = entrySet.iterator();
getAdminClient().streamingOps.updateSlopEntries(0, slopIterator);
// check updated values
Iterator<Versioned<Slop>> entrysetItr = entrySet.iterator();
while (entrysetItr.hasNext()) {
Versioned<Slop> versioned = entrysetItr.next();
Slop nextSlop = versioned.getValue();
Store<ByteArray, byte[], byte[]> store = getStore(0, nextSlop.getStoreName());
if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
} else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
}
}
}
use of voldemort.utils.ByteArray in project voldemort by voldemort.
the class SlopStreamingTest method testSlopStreaming.
/*
* Does streaming puts with one node marked down. Then we drain all slops
* and ensure all keys end up on the right nodes
*/
@Test
public void testSlopStreaming() throws InterruptedException {
testEnv = new SlopStreamingTestEnvironment(1, false);
// 3-2-2 setup
testEnv.setPreferredWrite(2).setRequiredWrite(2);
testEnv.start();
Set<Integer> nodeIds = new HashSet<Integer>();
long startMs = System.currentTimeMillis();
long endMs = startMs + MAX_TOTAL_TIME_MS;
long totalPuts = 0;
long numRejectedPuts = 0;
long numAssertPuts = 0;
streamer = testEnv.makeSlopStreamingClient();
while (true) {
if (System.currentTimeMillis() > endMs) {
break;
}
// generate key
ByteArray key = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
byte[] value = TestUtils.randomBytes(VALUE_LENGTH);
Versioned<byte[]> outputValue = Versioned.value(value);
// put to nodes
try {
// if put does not throw exception
List<Node> routes = testEnv.routeRequest(key.get());
for (Node node : routes) {
numAssertPuts++;
nodeIds.add(node.getId());
Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(node.getId());
if (realStore instanceof InMemoryPutAssertionStorageEngine) {
((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).assertPut(key);
} else {
fail("realStore is not InMemoryPutAssertionStorageEngine. Test setup is wrong");
}
}
streamer.streamingPut(key, outputValue, testEnv.STORE_NAME);
} catch (InsufficientOperationalNodesException e) {
numRejectedPuts++;
if (logger.isDebugEnabled()) {
logger.debug("Key " + key + " is rejected for InsufficientOperationalNodes");
}
} finally {
totalPuts++;
}
}
streamer.commitToVoldemort();
streamer.closeStreamingSessions();
// bring all servers up
testEnv.warpUp();
try {
streamer.commitToVoldemort();
streamer.closeStreamingSessions();
} catch (Exception e) {
logger.warn(e);
}
// check
long numFailedAssertions = 0;
for (Integer nodeId : nodeIds) {
Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(nodeId);
if (realStore instanceof InMemoryPutAssertionStorageEngine) {
Set<ByteArray> keys = ((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).getFailedAssertions();
for (ByteArray key : keys) {
logger.error("key [" + key + "] is asserted but not recorded on node [" + nodeId + "]");
}
numFailedAssertions += keys.size();
} else {
fail("realStore is not InMemoryPutAssertionStorageEngine");
}
}
logger.info("Total Client Puts Rejected (InsufficientOperationalNodes): " + numRejectedPuts);
logger.info("Total Client Put Operations: " + totalPuts);
logger.info("Total Server Put Assertions: " + numAssertPuts);
logger.info("Total Server Put Lost: " + numFailedAssertions);
assertEquals(numFailedAssertions + " server puts are lost: " + numFailedAssertions, 0L, numFailedAssertions);
}
use of voldemort.utils.ByteArray in project voldemort by voldemort.
the class SlopStreamingTestEnvironment method run.
@Override
public void run() {
Random random = new Random(System.currentTimeMillis());
cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
bootstrapUrl = cluster.getNodeById(0).getSocketUrl().toString();
storeDef = storeDefBuilder.build();
// setup store engines
for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
// do only once
createInnerStore(nodeId);
}
for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
try {
startServer(nodeId);
} catch (IOException e) {
logger.error("Server " + nodeId + "failed to start", e);
}
}
factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootstrapUrl));
// wait for start of servers
startFinishLatch.countDown();
try {
Set<Integer> nodesToFail = new HashSet<Integer>();
nodesToFail.add(1);
boolean wrapUpSignal = false;
if (logger.isInfoEnabled()) {
if (wrapUpSignal) {
logger.info("Wake Up and wrap up. Make all servers NORMAL");
} else {
logger.info("Wake Up and decide new failure statuses");
}
for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : realStores.entrySet()) {
InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]> engine = (InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) entry.getValue();
logger.info("Outstanding Put Assertions of node [" + entry.getKey() + "]: " + engine.getFailedAssertions().size());
}
}
if (logger.isInfoEnabled()) {
logger.info("Setting nodes to Fail: " + nodesToFail.toString());
}
for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
makeNodeNormal(nodeId);
}
for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
if (nodesToFail.contains(nodeId)) {
// fail a node if it's normal
if (nodesStatus.get(nodeId) == NodeStatus.NORMAL) {
makeNodeDown(nodeId);
}
} else {
// make node normal if not normal
if (nodesStatus.get(nodeId) != NodeStatus.NORMAL) {
makeNodeNormal(nodeId);
}
}
}
if (throwIntermittentException)
waitForFaultyNodeLatch.countDown();
if (!throwIntermittentException) {
for (Integer nodeId = 0; nodeId < NUM_NODES_TOTAL; nodeId++) {
makeNodeNormal(nodeId);
}
}
} catch (Exception e) {
} finally {
wrapUpFinishLatch.countDown();
}
}
use of voldemort.utils.ByteArray in project voldemort by voldemort.
the class AbstractAdminServiceFilterTest method testDeleteStreamWithFilter.
@Test
public void testDeleteStreamWithFilter() {
// user store should be present
Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
VoldemortFilter filter = new VoldemortFilterImpl();
RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
for (Pair<ByteArray, Versioned<byte[]>> pair : entrySet) {
if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0))
store.put(pair.getFirst(), pair.getSecond(), null);
}
// make delete stream call with filter
getAdminClient().storeMntOps.deletePartitions(0, testStoreName, Lists.newArrayList(0, 1), filter);
// assert none of the filtered entries are returned.
for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
if (Utils.nodeListToNodeIdList(strategy.routeRequest(entry.getFirst().get())).contains(0)) {
if (filter.accept(entry.getFirst(), entry.getSecond())) {
assertEquals("All entries should be deleted except the filtered ones.", 0, store.get(entry.getFirst(), null).size());
} else {
assertNotSame("filtered entry should be still present.", 0, store.get(entry.getFirst(), null).size());
assertEquals("values should match", new String(entry.getSecond().getValue()), new String(store.get(entry.getFirst(), null).get(0).getValue()));
}
}
}
}
use of voldemort.utils.ByteArray in project voldemort by voldemort.
the class AbstractAdminServiceFilterTest method testUpdateAsStreamWithFilter.
@Test
public void testUpdateAsStreamWithFilter() {
VoldemortFilter filter = new VoldemortFilterImpl();
Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
// make update stream call with filter
getAdminClient().streamingOps.updateEntries(0, testStoreName, entrySet.iterator(), filter);
// assert none of the filtered entries are updated.
// user store should be present
Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
if (filter.accept(entry.getFirst(), entry.getSecond())) {
assertEquals("Store should have this key/value pair", 1, store.get(entry.getFirst(), null).size());
assertEquals("Store should have this key/value pair", entry.getSecond(), store.get(entry.getFirst(), null).get(0));
} else {
assertEquals("Store should Not have this key/value pair", 0, store.get(entry.getFirst(), null).size());
}
}
}
Aggregations