Examples with StoreDefinition voldemort.store.StoreDefinition used on opensource projects

Example 6 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAllZZZ.

/**
 * Tests that getAll returns partial results in a 3 zone cluster (with a
 * node down).
 */
@Test
public void testPartialGetAllZZZ() throws Exception {
    // Set replication factors for a 3 zone cluster
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 1);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    // Create a store with RF=3, Required reads = 3 and zone count reads = 2
    // This ensures that a GET operation requires a response from all 3
    // nodes (from the respective 3 zones)
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).setReplicationFactor(3).setPreferredReads(3).setRequiredReads(3).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(2).setZoneCountWrites(1).setZoneReplicationFactor(zoneReplicationFactor).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        // Create zoned nodes - one in each zone (0, 1, 2)
        nodes.add(new Node(i, "none", 0, 0, 0, i, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    List<Zone> zones = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        LinkedList<Integer> zoneProximityList = Lists.newLinkedList();
        Set<Integer> zoneIds = Sets.newHashSet(0, 1, 2);
        zoneIds.remove(i);
        zoneProximityList.addAll(zoneIds);
        zones.add(new Zone(i, zoneProximityList));
    }
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes, zones), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    // Expected
    }
}

Example 7 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAll.

/**
 * Tests that getAll returns partial results
 */
@Test
public void testPartialGetAll() throws Exception {
    // create a store with rf=1 i.e disjoint partitions
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    }
}

Example 8 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testGetAllWithFailingStore.

@Test
public void testGetAllWithFailingStore() throws Exception {
    cluster = VoldemortTestConstants.getTwoNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 2, 1, 1, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int id1 = Iterables.get(cluster.getNodes(), 0).getId();
    int id2 = Iterables.get(cluster.getNodes(), 1).getId();
    subStores.put(id1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    subStores.put(id2, new FailingReadsStore<ByteArray, byte[], byte[]>("test"));
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        store.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    Map<ByteArray, List<Versioned<byte[]>>> all = store.getAll(expectedValues.keySet(), null);
    assertEquals(expectedValues.size(), all.size());
    for (Map.Entry<ByteArray, List<Versioned<byte[]>>> mapEntry : all.entrySet()) {
        byte[] value = expectedValues.get(mapEntry.getKey());
        assertEquals(new ByteArray(value), new ByteArray(mapEntry.getValue().get(0).getValue()));
    }
}

Example 9 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testGetAllWithMorePreferredReadsThanNodes.

/**
 * One node up, two preferred reads and one required read. See:
 *
 * http://github.com/voldemort/voldemort/issues#issue/18
 */
@Test
public void testGetAllWithMorePreferredReadsThanNodes() throws Exception {
    cluster = VoldemortTestConstants.getTwoNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 2, 2, 1, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int id1 = Iterables.get(cluster.getNodes(), 0).getId();
    int id2 = Iterables.get(cluster.getNodes(), 1).getId();
    subStores.put(id1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    subStores.put(id2, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    store.put(aKey, Versioned.value(aValue), aTransform);
    recordException(failureDetector, cluster.getNodes().iterator().next());
    Map<ByteArray, List<Versioned<byte[]>>> all = store.getAll(Arrays.asList(aKey), Collections.singletonMap(aKey, aTransform));
    assertEquals(1, all.size());
    assertTrue(Arrays.equals(aValue, all.values().iterator().next().get(0).getValue()));
}

Example 10 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testPutDeleteZoneRouting.

@Test
public void testPutDeleteZoneRouting() throws Exception {
    cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 2);
    Versioned<byte[]> versioned = new Versioned<byte[]>(new byte[] { 1 });
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    Set<Integer> sleepy = Sets.newHashSet(4, 5, 6, 7);
    for (Node n : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> subStore = null;
        if (sleepy != null && sleepy.contains(n.getId()))
            subStore = new SleepyStore<ByteArray, byte[], byte[]>(SLEEPY_TIME, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        else
            subStore = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test");
        subStores.put(n.getId(), subStore);
    }
    setFailureDetector(subStores);
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 1, 1, 1, 1, 0, 0, zoneReplicationFactor, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    routedStoreThreadPool = Executors.newFixedThreadPool(8);
    RoutedStoreFactory routedStoreFactory = createFactory();
    Store<ByteArray, byte[], byte[]> s1 = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(OPERATION_TIMEOUT));
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    List<Node> nodesRoutedTo = routingStrategy.routeRequest("test".getBytes());
    long start = System.nanoTime(), elapsed;
    try {
        s1.put(new ByteArray("test".getBytes()), versioned, null);
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).get(0), versioned);
    }
    // make sure the failure detector adds back any previously failed nodes
    Thread.sleep(BANNAGE_PERIOD + 100);
    start = System.nanoTime();
    try {
        s1.delete(new ByteArray("test".getBytes()), versioned.getVersion());
    } finally {
        elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + SLEEPY_TIME, elapsed < SLEEPY_TIME);
    }
    Thread.sleep(SLEEPY_TIME - elapsed);
    for (Node node : nodesRoutedTo) {
        assertEquals(subStores.get(node.getId()).get(new ByteArray("test".getBytes()), null).size(), 0);
    }
}