Examples with SleepyStore voldemort.store.SleepyStore used on opensource projects

Example 6 with SleepyStore

use of voldemort.store.SleepyStore in project voldemort by voldemort.

the class RoutedStoreTest method testPutDeleteZoneRouting.

@Test
public void testPutDeleteZoneRouting() throws Exception {
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 2);
    Versioned<byte[]> versioned = new Versioned<byte[]>(new byte[] { 1 });
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    Set<Integer> sleepy = Sets.newHashSet(4, 5, 6, 7);
    for (Node n : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> subStore = null;
        if (sleepy != null && sleepy.contains(n.getId()))
            subStore = new SleepyStore<ByteArray, byte[], byte[]>(SLEEPY_TIME, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        else
            subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
    }
    setFailureDetector(subStores);
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 1, 1, 1, 1, 0, 0, zoneReplicationFactor, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    routedStoreThreadPool = Executors.newFixedThreadPool(8);
    RoutedStoreFactory routedStoreFactory = createFactory();
    Store<ByteArray, byte[], byte[]> s1 = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(OPERATION_TIMEOUT));
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    List<Node> nodesRoutedTo = routingStrategy.routeRequest("test".getBytes());
    long start = System.nanoTime(), elapsed;
    try {
        s1.put(new ByteArray("test".getBytes()), versioned, null);
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).get(0), versioned);
    }
    // make sure the failure detector adds back any previously failed nodes
    Thread.sleep(BANNAGE_PERIOD + 100);
    start = System.nanoTime();
    try {
        s1.delete(new ByteArray("test".getBytes()), versioned.getVersion());
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).size(), 0);
    }
}

Example 7 with SleepyStore

use of voldemort.store.SleepyStore in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAll.

/**
     * Tests that getAll returns partial results
     */
@Test
public void testPartialGetAll() throws Exception {
    // create a store with rf=1 i.e disjoint partitions
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    }
}

Example 8 with SleepyStore

use of voldemort.store.SleepyStore in project voldemort by voldemort.

the class RoutedStoreTest method testGetAndPutTimeoutZZZ.

@Test
public void testGetAndPutTimeoutZZZ() throws Exception {
    int timeout = 50;
    // Set replication factors for a 3 zone cluster
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 1);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    // Create a store with RF=3, Required reads = 3 and zone count reads = 2
    // This ensures that a GET operation requires a response from all 3
    // nodes (from the respective 3 zones)
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).setReplicationFactor(3).setPreferredReads(3).setRequiredReads(3).setPreferredWrites(3).setRequiredWrites(3).setZoneCountReads(2).setZoneCountWrites(2).setZoneReplicationFactor(zoneReplicationFactor).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    int totalDelay = 0;
    for (int i = 0; i < 3; i++) {
        int delay = 4 + i * timeout;
        totalDelay += delay;
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(delay, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, i, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    RoutedStoreFactory routedStoreFactory = createFactory();
    List<Zone> zones = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        LinkedList<Integer> zoneProximityList = Lists.newLinkedList();
        Set<Integer> zoneIds = Sets.newHashSet(0, 1, 2);
        zoneIds.remove(i);
        zoneProximityList.addAll(zoneIds);
        zones.add(new Zone(i, zoneProximityList));
    }
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes, zones), definition, stores, failureDetector, createConfig(timeout));
    long start = System.nanoTime();
    try {
        routedStore.get(new ByteArray("test".getBytes()), null);
        fail("Should have thrown");
    } catch (InsufficientOperationalNodesException e) {
        long elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + totalDelay, elapsed < totalDelay);
    }
    start = System.nanoTime();
    try {
        routedStore.put(new ByteArray("test".getBytes()), new Versioned<byte[]>(new byte[] { 1 }), null);
        fail("Should have thrown");
    } catch (InsufficientOperationalNodesException e) {
        long elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + totalDelay, elapsed < totalDelay);
    }
}

Example 9 with SleepyStore

use of voldemort.store.SleepyStore in project voldemort by voldemort.

the class HintedHandoffTestEnvironment method createInnerStore.

/**
     * Create inner store and storage engines before server starts
     * 
     * @param nodeId
     */
public void createInnerStore(int nodeId) {
    Store<ByteArray, byte[], byte[]> realStore = new InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
    ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = new ForceFailStore<ByteArray, byte[], byte[]>(realStore, new PersistenceFailureException("Force failed"));
    SleepyStore<ByteArray, byte[], byte[]> sleepyStore = new SleepyStore<ByteArray, byte[], byte[]>(0, forceFailStore);
    realStores.put(nodeId, realStore);
    forceFailStores.put(nodeId, forceFailStore);
    sleepyStores.put(nodeId, sleepyStore);
}

Example 10 with SleepyStore

use of voldemort.store.SleepyStore in project voldemort by voldemort.

the class HintedHandoffFailureTest method customSetup.

/**
     * Setup a cluster with 3 nodes, with the following characteristics:
     * 
     * If FAILURE_MODE is FAIL_FIRST_REPLICA_NODE set the first replica store to
     * a sleepy force failing store
     * 
     * If FAILURE_MODE is FAIL_ALL_REPLICAS: set all replicas to sleepy force
     * failing store
     * 
     * Pseudo master : Standard In-memory store (wrapped by Logging store)
     * 
     * In memory slop stores
     * 
     * @param key The ByteArray representation of the key
     * @param failureMode The Failure mode for the replicas
     * 
     * @throws Exception
     */
public List<Integer> customSetup(ByteArray key, FAILURE_MODE failureMode, ReplicaFactor replicaFactor, long sleepTime, long hintDelayTimeMs) throws Exception {
    cluster = getThreeNodeCluster();
    storeDef = getStoreDef(STORE_NAME, replicaFactor, RoutingStrategyType.CONSISTENT_STRATEGY);
    strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    InMemoryStorageEngine<ByteArray, byte[], byte[]> inMemoryStorageEngine = new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME);
    LoggingStore<ByteArray, byte[], byte[]> loggingStore = new LoggingStore<ByteArray, byte[], byte[]>(inMemoryStorageEngine);
    VoldemortException e = new UnreachableStoreException("Node down");
    ForceFailStore<ByteArray, byte[], byte[]> failureStore = new ForceFailStore<ByteArray, byte[], byte[]>(loggingStore, e);
    SleepyStore<ByteArray, byte[], byte[]> sleepyFailureStore = new SleepyStore<ByteArray, byte[], byte[]>(sleepTime, failureStore);
    failureStore.setFail(true);
    List<Integer> failingNodeIdList = Lists.newArrayList();
    List<Node> replicaList = strategy.routeRequest(key.get());
    switch(failureMode) {
        case FAIL_FIRST_REPLICA_NODE:
            failingNodeIdList.add(replicaList.get(1).getId());
            break;
        case FAIL_ALL_REPLICAS:
            for (int nodeId = 1; nodeId < replicaList.size(); nodeId++) {
                failingNodeIdList.add(nodeId);
            }
            break;
    }
    subStores.clear();
    for (int i = 0; i < NUM_NODES_TOTAL; i++) {
        if (failingNodeIdList.contains(i)) {
            subStores.put(i, sleepyFailureStore);
        } else {
            subStores.put(i, loggingStore);
        }
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(NUM_THREADS);
    routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    for (Node node : cluster.getNodes()) {
        int nodeId = node.getId();
        SlopStorageEngine slopStorageEngine = new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>(SLOP_STORE_NAME), cluster);
        StorageEngine<ByteArray, Slop, byte[]> storageEngine = slopStorageEngine.asSlopStore();
        nonblockingSlopStores.put(nodeId, routedStoreFactory.toNonblockingStore(slopStorageEngine));
        slopStores.put(nodeId, storageEngine);
    }
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> entry : subStores.entrySet()) nonblockingStores.put(entry.getKey(), routedStoreFactory.toNonblockingStore(entry.getValue()));
    store = new DelayedPutPipelineRoutedStore(subStores, nonblockingStores, slopStores, nonblockingSlopStores, cluster, storeDef, failureDetector, hintDelayTimeMs);
    return failingNodeIdList;
}