Examples with Slop voldemort.store.slop.Slop used on opensource projects

Example 31 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class HintedHandoffFailureTest method testNoSlopsOnAllReplicaFailures.

/**
     * Test to do a put with a 3-2-2 config such that both the replica nodes do
     * not respond at all. This test is to make sure that the main thread
     * returns with an error and that no slops are registered.
     */
@Test
public void testNoSlopsOnAllReplicaFailures() throws Exception {
    String key = "testNoSlopsOnAllReplicaFailures";
    String val = "xyz";
    final Versioned<byte[]> versionedVal = new Versioned<byte[]>(val.getBytes());
    final ByteArray keyByteArray = new ByteArray(key.getBytes());
    List<Integer> failingNodeIdList = null;
    long sleepTime = ROUTING_TIMEOUT_MS + 100;
    failingNodeIdList = customSetup(keyByteArray, FAILURE_MODE.FAIL_ALL_REPLICAS, get322Replica(), sleepTime, 0);
    PerformAsyncPut asyncPutThread = new PerformAsyncPut(this.store, keyByteArray, versionedVal);
    Executors.newFixedThreadPool(1).submit(asyncPutThread);
    // Sleep for the routing timeout with some headroom
    try {
        Thread.sleep(sleepTime + 100);
    } catch (Exception e) {
        fail("Unknown error while doing a put: " + e);
    }
    // Check the slop stores
    Set<ByteArray> failedKeys = Sets.newHashSet();
    failedKeys.add(keyByteArray);
    Set<ByteArray> slopKeys = makeSlopKeys(keyByteArray, failingNodeIdList);
    Set<Slop> registeredSlops = getAllSlops(slopKeys);
    if (registeredSlops.size() != 0) {
        fail("Should not have seen any slops.");
    }
}

Example 32 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class HintedHandoffFailureTest method testSlopOnDelayedFailingAsyncPut_3_2_2.

/**
     * Test to ensure that when an asynchronous put completes (with a failure)
     * after PerformParallelPut has finished processing the responses and before
     * the hinted handoff actually begins, a slop is still registered for the
     * same.
     * 
     * This is for the 3-2-2 configuration.
     */
@Test
public void testSlopOnDelayedFailingAsyncPut_3_2_2() throws Exception {
    String key = "testSlopOnDelayedFailingAsyncPut_3_2_2";
    String val = "xyz";
    Versioned<byte[]> versionedVal = new Versioned<byte[]>(val.getBytes());
    ByteArray keyByteArray = new ByteArray(key.getBytes());
    List<Integer> failingNodeIdList = null;
    try {
        failingNodeIdList = customSetup(keyByteArray, get322Replica(), HINT_DELAY_TIME_MS);
    } catch (Exception e) {
        logger.info(e.getMessage());
        fail("Error in setup.");
    }
    this.store.put(keyByteArray, versionedVal, null);
    Thread.sleep(HINT_DELAY_TIME_MS + 100);
    // Check the slop stores
    Set<ByteArray> failedKeys = Sets.newHashSet();
    failedKeys.add(keyByteArray);
    Set<ByteArray> slopKeys = makeSlopKeys(keyByteArray, failingNodeIdList);
    Set<Slop> registeredSlops = getAllSlops(slopKeys);
    if (registeredSlops.size() == 0) {
        fail("Should have seen some slops. But could not find any.");
    } else if (registeredSlops.size() != 1) {
        fail("Number of slops registered != 1");
    }
}

Example 33 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class HintedHandoffFailureTest method customSetup.

/**
     * Setup a cluster with 3 nodes, with the following characteristics:
     * 
     * If FAILURE_MODE is FAIL_FIRST_REPLICA_NODE set the first replica store to
     * a sleepy force failing store
     * 
     * If FAILURE_MODE is FAIL_ALL_REPLICAS: set all replicas to sleepy force
     * failing store
     * 
     * Pseudo master : Standard In-memory store (wrapped by Logging store)
     * 
     * In memory slop stores
     * 
     * @param key The ByteArray representation of the key
     * @param failureMode The Failure mode for the replicas
     * 
     * @throws Exception
     */
public List<Integer> customSetup(ByteArray key, FAILURE_MODE failureMode, ReplicaFactor replicaFactor, long sleepTime, long hintDelayTimeMs) throws Exception {
    cluster = getThreeNodeCluster();
    storeDef = getStoreDef(STORE_NAME, replicaFactor, RoutingStrategyType.CONSISTENT_STRATEGY);
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryStorageEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
    LoggingStore<ByteArray, byte[], byte[]> loggingStore = new LoggingStore<ByteArray, byte[], byte[]>(inMemoryStorageEngine);
    VoldemortException e = new UnreachableStoreException("Node down");
    ForceFailStore<ByteArray, byte[], byte[]> failureStore = new ForceFailStore<ByteArray, byte[], byte[]>(loggingStore, e);
    SleepyStore<ByteArray, byte[], byte[]> sleepyFailureStore = new SleepyStore<ByteArray, byte[], byte[]>(sleepTime, failureStore);
    failureStore.setFail(true);
    List<Integer> failingNodeIdList = Lists.newArrayList();
    List<Node> replicaList = strategy.routeRequest(key.get());
    switch(failureMode) {
        case FAIL_FIRST_REPLICA_NODE:
            failingNodeIdList.add(replicaList.get(1).getId());
            break;
        case FAIL_ALL_REPLICAS:
            for (int nodeId = 1; nodeId < replicaList.size(); nodeId++) {
                failingNodeIdList.add(nodeId);
            }
            break;
    }
    subStores.clear();
    for (int i = 0; i < NUM_NODES_TOTAL; i++) {
        if (failingNodeIdList.contains(i)) {
            subStores.put(i, sleepyFailureStore);
        } else {
            subStores.put(i, loggingStore);
        }
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(NUM_THREADS);
    routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    for (Node node : cluster.getNodes()) {
        int nodeId = node.getId();
        SlopStorageEngine slopStorageEngine = new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(SLOP_STORE_NAME), cluster);
        StorageEngine<ByteArray, Slop, byte[]> storageEngine = slopStorageEngine.asSlopStore();
        nonblockingSlopStores.put(nodeId, routedStoreFactory.toNonblockingStore(slopStorageEngine));
        slopStores.put(nodeId, storageEngine);
    }
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : subStores.entrySet()) nonblockingStores.put(entry.getKey(), routedStoreFactory.toNonblockingStore(entry.getValue()));
    store = new DelayedPutPipelineRoutedStore(subStores, nonblockingStores, slopStores, nonblockingSlopStores, cluster, storeDef, failureDetector, hintDelayTimeMs);
    return failingNodeIdList;
}

Example 34 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class HintedHandoffFailureTest method getAllSlops.

/**
     * A function to fetch all the registered slops
     * 
     * @param slopKeys Keys for the registered slops in the slop store
     * @return Set of all the registered Slops
     */
public Set<Slop> getAllSlops(Iterable<ByteArray> slopKeys) {
    Set<Slop> registeredSlops = Sets.newHashSet();
    for (Store<ByteArray, Slop, byte[]> slopStore : slopStores.values()) {
        Map<ByteArray, List<Versioned<Slop>>> res = slopStore.getAll(slopKeys, null);
        for (Map.Entry<ByteArray, List<Versioned<Slop>>> entry : res.entrySet()) {
            Slop slop = entry.getValue().get(0).getValue();
            registeredSlops.add(slop);
            logger.info(slop);
        }
    }
    return registeredSlops;
}

Example 35 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class HintedHandoffFailureTest method testSlopViaSerialHint_2_1_1.

/**
     * Test to ensure that when an asynchronous put completes (with a failure)
     * after the pipeline completes, a slop is still registered (via a serial
     * hint).
     * 
     * This is for the 2-1-1 configuration
     */
@Test
public void testSlopViaSerialHint_2_1_1() {
    String key = "testSlopViaSerialHint_2_1_1";
    String val = "xyz";
    Versioned<byte[]> versionedVal = new Versioned<byte[]>(val.getBytes());
    ByteArray keyByteArray = new ByteArray(key.getBytes());
    List<Integer> failingNodeIdList = null;
    try {
        failingNodeIdList = customSetup(keyByteArray, get211Replica(), 0);
    } catch (Exception e) {
        logger.info(e.getMessage());
        fail("Error in setup.");
    }
    this.store.put(keyByteArray, versionedVal, null);
    // Give enough time for the serial hint to work.
    try {
        logger.info("Sleeping for 5 seconds to wait for the serial hint to finish");
        Thread.sleep(1000);
    } catch (Exception e) {
    }
    // Check the slop stores
    Set<ByteArray> failedKeys = Sets.newHashSet();
    failedKeys.add(keyByteArray);
    Set<ByteArray> slopKeys = makeSlopKeys(keyByteArray, failingNodeIdList);
    Set<Slop> registeredSlops = getAllSlops(slopKeys);
    if (registeredSlops.size() == 0) {
        fail("Should have seen some slops. But could not find any.");
    } else if (registeredSlops.size() != 1) {
        fail("Number of slops registered != 1");
    }
}