Examples with InconsistencyResolvingStore voldemort.store.versioned.InconsistencyResolvingStore used on opensource projects

Example 6 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testGetAllWithFailingStoreZZZ.

/**
     * Test to ensure that get all works in a 3 zone cluster with 2 nodes per
     * zone and 1 node down in each zone.
     */
@Test
public void testGetAllWithFailingStoreZZZ() throws Exception {
    cluster = VoldemortTestConstants.getSixNodeClusterWith3Zones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(1, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(2, cluster.getNumberOfNodesInZone(0));
    // PR = RR = 3
    // PW = RW = 6
    // Zone Reads = 2
    // Zone Writes = 2
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 3, 3, 6, 6, 2, 2, zoneReplicationFactor, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    for (int id = 0; id < 6; id++) {
        // other is normal
        if (id % 2 == 0) {
            subStores.put(id, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        } else {
            subStores.put(id, new FailingReadsStore<ByteArray, byte[], byte[]>("test"));
        }
    }
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        store.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    Map<ByteArray, List<Versioned<byte[]>>> all = store.getAll(expectedValues.keySet(), null);
    assertEquals(expectedValues.size(), all.size());
    for (Map.Entry<ByteArray, List<Versioned<byte[]>>> mapEntry : all.entrySet()) {
        byte[] value = expectedValues.get(mapEntry.getKey());
        assertEquals(new ByteArray(value), new ByteArray(mapEntry.getValue().get(0).getValue()));
    }
}

Example 7 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testGetAllWithFailingStore.

@Test
public void testGetAllWithFailingStore() throws Exception {
    cluster = VoldemortTestConstants.getTwoNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 2, 1, 1, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int id1 = Iterables.get(cluster.getNodes(), 0).getId();
    int id2 = Iterables.get(cluster.getNodes(), 1).getId();
    subStores.put(id1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    subStores.put(id2, new FailingReadsStore<ByteArray, byte[], byte[]>("test"));
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        store.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    Map<ByteArray, List<Versioned<byte[]>>> all = store.getAll(expectedValues.keySet(), null);
    assertEquals(expectedValues.size(), all.size());
    for (Map.Entry<ByteArray, List<Versioned<byte[]>>> mapEntry : all.entrySet()) {
        byte[] value = expectedValues.get(mapEntry.getKey());
        assertEquals(new ByteArray(value), new ByteArray(mapEntry.getValue().get(0).getValue()));
    }
}

Example 8 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testReadRepairWithFailuresZZZ.

/**
     * Test to ensure that read repair happens correctly across zones in case of
     * inconsistent writes in a 3 zone cluster.
     * 
     * @throws Exception
     */
@Test
public void testReadRepairWithFailuresZZZ() throws Exception {
    cluster = VoldemortTestConstants.getSixNodeClusterWith3Zones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(1, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(2, cluster.getNumberOfNodesInZone(0));
    // PR = RR = 6
    // PW = RW = 4
    // Zone Reads = # Zones - 1
    // Zone Writes = 1
    // Threads = 1
    RoutedStore routedStore = getStore(cluster, 6, 4, cluster.getNumberOfZones() - 1, 1, 1, zoneReplicationFactor);
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    BaseStoreRoutingPlan routingPlan = new BaseStoreRoutingPlan(cluster, this.storeDef);
    List<Integer> replicatingNodes = routingPlan.getReplicationNodeList(aKey.get());
    try {
        // Do the initial put with all nodes up
        store.put(aKey, new Versioned<byte[]>(aValue), null);
        List<Version> initialVersions = store.getVersions(aKey);
        assertEquals(6, initialVersions.size());
        Version mainVersion = initialVersions.get(0);
        for (int i = 1; i < initialVersions.size(); i++) {
            assertEquals(mainVersion, initialVersions.get(i));
        }
        // Do another put with all nodes in the zone 0 marked as
        // unavailable. This will force the put to use a different pseudo
        // master than before.
        byte[] anotherValue = "john".getBytes();
        // In this cluster, nodes 0 and 1 are in Zone 0. Mark them
        // unavailable
        recordException(failureDetector, cluster.getNodeById(0));
        recordException(failureDetector, cluster.getNodeById(1));
        Version newVersion = ((VectorClock) mainVersion).clone();
        store.put(aKey, new Versioned<byte[]>(anotherValue, newVersion), null);
        waitForOperationToComplete(500);
        // Mark the nodes in Zone 0 as available and do a get. The Required
        // reads = 4 and Zone count reads = 2 will force the client to read
        // from all the zones and do the essential read repairs.
        recordSuccess(failureDetector, cluster.getNodeById(0));
        recordSuccess(failureDetector, cluster.getNodeById(1));
        List<Versioned<byte[]>> versioneds = store.get(aKey, null);
        assertEquals(1, versioneds.size());
        assertEquals(new ByteArray(anotherValue), new ByteArray(versioneds.get(0).getValue()));
        // Read repairs are done asynchronously, so we sleep for a short
        // period. It may be a good idea to use a synchronous executor
        // service.
        Thread.sleep(500);
        for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> innerStoreEntry : routedStore.getInnerStores().entrySet()) {
            // Only look at the nodes in the pref list
            if (replicatingNodes.contains(innerStoreEntry.getKey())) {
                List<Versioned<byte[]>> innerVersioneds = innerStoreEntry.getValue().get(aKey, null);
                assertEquals(1, versioneds.size());
                assertEquals(new ByteArray(anotherValue), new ByteArray(innerVersioneds.get(0).getValue()));
            }
        }
    } catch (VoldemortException ve) {
        fail("Unexpected error occurred : " + ve);
    }
}

Example 9 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testPutWithOneNodeDownAndOneNodeSlow.

/**
     * See issue #134: RoutedStore put() doesn't wait for enough attempts to
     * succeed
     * 
     * This issue would only happen with one node down and another that was slow
     * to respond.
     */
@Test
public void testPutWithOneNodeDownAndOneNodeSlow() throws Exception {
    cluster = VoldemortTestConstants.getThreeNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 3, 2, 2, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    /* The key used causes the nodes selected for writing to be [2, 0, 1] */
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int id1 = Iterables.get(cluster.getNodes(), 0).getId();
    int id2 = Iterables.get(cluster.getNodes(), 1).getId();
    int id3 = Iterables.get(cluster.getNodes(), 2).getId();
    subStores.put(id3, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    subStores.put(id1, new FailingStore<ByteArray, byte[], byte[]>("test"));
    /*
         * The bug would only show itself if the second successful required
         * write was slow (but still within the timeout).
         */
    subStores.put(id2, new SleepyStore<ByteArray, byte[], byte[]>(100, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test")));
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    store.put(aKey, new Versioned<byte[]>(aValue), aTransform);
}

Example 10 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class MockStoreClientFactory method getRawStore.

private <K1, V1, T1> Store<K1, V1, T1> getRawStore(String storeName) {
    List<StoreDefinition> storeDefs = storeMapper.readStoreList(new StringReader(storesXml));
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null)
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    DefaultSerializerFactory serializerFactory = new DefaultSerializerFactory();
    Serializer<K1> keySerializer = (Serializer<K1>) serializerFactory.getSerializer(storeDef.getKeySerializer());
    Serializer<V1> valueSerializer = (Serializer<V1>) serializerFactory.getSerializer(storeDef.getValueSerializer());
    Serializer<T1> transformsSerializer = null;
    if (storeDef.isView())
        transformsSerializer = (Serializer<T1>) serializerFactory.getSerializer(storeDef.getTransformsSerializer());
    // Add inconsistency resolving decorator, using their inconsistency
    // resolver (if they gave us one)
    InconsistencyResolver<Versioned<V1>> secondaryResolver = new TimeBasedInconsistencyResolver();
    StorageEngine engine;
    if (storeDef.isView())
        engine = new InMemoryStorageEngine(storeDef.getViewTargetStoreName());
    else
        engine = new InMemoryStorageEngine(storeDef.getName());
    if (storeDef.isView()) {
        // instantiate view
        String targetName = storeDef.getViewTargetStoreName();
        StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
        engine = new ViewStorageEngine(storeName, engine, this.viewValueSerializer != null ? this.viewValueSerializer : serializerFactory.getSerializer(storeDef.getValueSerializer()), this.transformsSerializer != null ? this.transformsSerializer : serializerFactory.getSerializer(storeDef.getTransformsSerializer()), this.keySerializer != null ? this.keySerializer : serializerFactory.getSerializer(targetDef.getKeySerializer()), this.valueSerializer != null ? this.valueSerializer : serializerFactory.getSerializer(targetDef.getValueSerializer()), null, ViewStorageConfiguration.loadTransformation(storeDef.getValueTransformation()));
    }
    Store store = new VersionIncrementingStore(engine, nodeId, time);
    store = new SerializingStore(store, this.keySerializer != null ? this.keySerializer : keySerializer, this.valueSerializer != null ? this.valueSerializer : valueSerializer, this.transformsSerializer != null ? this.transformsSerializer : transformsSerializer);
    Store<K1, V1, T1> consistentStore = new InconsistencyResolvingStore<K1, V1, T1>(store, new ChainedResolver<Versioned<V1>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    return consistentStore;
}