Examples with SerializerDefinition voldemort.serialization.SerializerDefinition used on opensource projects

Example 21 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testGetAndPutTimeoutZZZ.

@Test
public void testGetAndPutTimeoutZZZ() throws Exception {
    int timeout = 50;
    // Set replication factors for a 3 zone cluster
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 1);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    // Create a store with RF=3, Required reads = 3 and zone count reads = 2
    // This ensures that a GET operation requires a response from all 3
    // nodes (from the respective 3 zones)
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).setReplicationFactor(3).setPreferredReads(3).setRequiredReads(3).setPreferredWrites(3).setRequiredWrites(3).setZoneCountReads(2).setZoneCountWrites(2).setZoneReplicationFactor(zoneReplicationFactor).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    int totalDelay = 0;
    for (int i = 0; i < 3; i++) {
        int delay = 4 + i * timeout;
        totalDelay += delay;
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(delay, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, i, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    RoutedStoreFactory routedStoreFactory = createFactory();
    List<Zone> zones = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        LinkedList<Integer> zoneProximityList = Lists.newLinkedList();
        Set<Integer> zoneIds = Sets.newHashSet(0, 1, 2);
        zoneIds.remove(i);
        zoneProximityList.addAll(zoneIds);
        zones.add(new Zone(i, zoneProximityList));
    }
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes, zones), definition, stores, failureDetector, createConfig(timeout));
    long start = System.nanoTime();
    try {
        routedStore.get(new ByteArray("test".getBytes()), null);
        fail("Should have thrown");
    } catch (InsufficientOperationalNodesException e) {
        long elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + totalDelay, elapsed < totalDelay);
    }
    start = System.nanoTime();
    try {
        routedStore.put(new ByteArray("test".getBytes()), new Versioned<byte[]>(new byte[] { 1 }), null);
        fail("Should have thrown");
    } catch (InsufficientOperationalNodesException e) {
        long elapsed = (System.nanoTime() - start) / Time.NS_PER_MS;
        assertTrue(elapsed + " < " + totalDelay, elapsed < totalDelay);
    }
}

Example 22 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAllZZZ.

/**
 * Tests that getAll returns partial results in a 3 zone cluster (with a
 * node down).
 */
@Test
public void testPartialGetAllZZZ() throws Exception {
    // Set replication factors for a 3 zone cluster
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 1);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    // Create a store with RF=3, Required reads = 3 and zone count reads = 2
    // This ensures that a GET operation requires a response from all 3
    // nodes (from the respective 3 zones)
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).setReplicationFactor(3).setPreferredReads(3).setRequiredReads(3).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(2).setZoneCountWrites(1).setZoneReplicationFactor(zoneReplicationFactor).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        // Create zoned nodes - one in each zone (0, 1, 2)
        nodes.add(new Node(i, "none", 0, 0, 0, i, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    List<Zone> zones = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        LinkedList<Integer> zoneProximityList = Lists.newLinkedList();
        Set<Integer> zoneIds = Sets.newHashSet(0, 1, 2);
        zoneIds.remove(i);
        zoneProximityList.addAll(zoneIds);
        zones.add(new Zone(i, zoneProximityList));
    }
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes, zones), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    // Expected
    }
}

Example 23 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAll.

/**
 * Tests that getAll returns partial results
 */
@Test
public void testPartialGetAll() throws Exception {
    // create a store with rf=1 i.e disjoint partitions
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    }
}

Example 24 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class GetallNodeReachTest method testGetall_211_zoneCountRead_1.

@Test
public void testGetall_211_zoneCountRead_1() throws Exception {
    cluster = getFourNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = new HashMap<Integer, Integer>();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    /*
         * First n nodes on the preference list will be one node from each
         * remote n zones, where n=zoneCountReads, therefore preferred read
         * should be set > n if want to include local zone node results in
         * parallel request
         */
    storeDef = new StoreDefinitionBuilder().setName("test").setType(InMemoryStorageConfiguration.TYPE_NAME).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setZoneReplicationFactor(zoneReplicationFactor).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setPreferredReads(2).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(1).setZoneCountWrites(0).build();
    makeStore();
    Versioned<byte[]> v = Versioned.value("v".getBytes());
    subStores.get(0).put(TestUtils.toByteArray("k001"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k011"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k101"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k111"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k010"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k011"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k110"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k111"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k100"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k101"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k110"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k111"), v, null);
    /* test multiple keys getall */
    List<ByteArray> keys = new ArrayList<ByteArray>();
    keys.add(TestUtils.toByteArray("k000"));
    keys.add(TestUtils.toByteArray("k001"));
    keys.add(TestUtils.toByteArray("k010"));
    keys.add(TestUtils.toByteArray("k011"));
    keys.add(TestUtils.toByteArray("k100"));
    keys.add(TestUtils.toByteArray("k101"));
    keys.add(TestUtils.toByteArray("k110"));
    keys.add(TestUtils.toByteArray("k111"));
    Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keys, null);
    assertFalse(result.containsKey(TestUtils.toByteArray("not_included")));
    /* client will first try all the nodes in local zone */
    assertFalse(result.containsKey(TestUtils.toByteArray("k000")));
    assertEquals(1, result.get(TestUtils.toByteArray("k011")).size());
    assertFalse(result.containsKey(TestUtils.toByteArray("not_included")));
    assertFalse(result.containsKey(TestUtils.toByteArray("k000")));
    assertEquals(1, result.get(TestUtils.toByteArray("k011")).size());
    assertEquals(1, result.get(TestUtils.toByteArray("k100")).size());
    assertEquals(2, result.get(TestUtils.toByteArray("k111")).size());
}

Example 25 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class GetallNodeReachTest method testGetall_211.

@Test
public void testGetall_211() throws Exception {
    cluster = getFourNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = new HashMap<Integer, Integer>();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    storeDef = new StoreDefinitionBuilder().setName("test").setType(InMemoryStorageConfiguration.TYPE_NAME).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setZoneReplicationFactor(zoneReplicationFactor).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).build();
    makeStore();
    Versioned<byte[]> v = Versioned.value("v".getBytes());
    // k### indicates existence of itself in different nodes
    // k**1 means this key exists at least on node 0
    // k*1* means this key exists at least on node 1
    // k0** means this key does not exist on node 2
    subStores.get(0).put(TestUtils.toByteArray("k001"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k011"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k101"), v, null);
    subStores.get(0).put(TestUtils.toByteArray("k111"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k010"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k011"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k110"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k111"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k100"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k101"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k110"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k111"), v, null);
    /* test multiple keys getall */
    List<ByteArray> keys = new ArrayList<ByteArray>();
    keys.add(TestUtils.toByteArray("k000"));
    keys.add(TestUtils.toByteArray("k001"));
    keys.add(TestUtils.toByteArray("k010"));
    keys.add(TestUtils.toByteArray("k011"));
    keys.add(TestUtils.toByteArray("k100"));
    keys.add(TestUtils.toByteArray("k101"));
    keys.add(TestUtils.toByteArray("k110"));
    keys.add(TestUtils.toByteArray("k111"));
    Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keys, null);
    assertFalse(result.containsKey(TestUtils.toByteArray("not_included")));
    assertFalse(result.containsKey(TestUtils.toByteArray("k000")));
    assertEquals(1, result.get(TestUtils.toByteArray("k011")).size());
    assertFalse(result.containsKey(TestUtils.toByteArray("k100")));
    assertEquals(1, result.get(TestUtils.toByteArray("k111")).size());
}