Examples with InconsistencyResolvingStore voldemort.store.versioned.InconsistencyResolvingStore used on opensource projects

Example 1 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class AbstractStoreClientFactory method getRawStore.

@SuppressWarnings("unchecked")
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver, String customStoresXml, String clusterXmlString, FailureDetector fd) {
    logger.info("Client zone-id [" + this.routedStoreConfig.getClientZoneId() + "] Attempting to get raw store [" + storeName + "] ");
    if (logger.isDebugEnabled()) {
        for (URI uri : bootstrapUrls) {
            logger.debug("Client Bootstrap url [" + uri + "]");
        }
    }
    // Get cluster and store metadata
    String clusterXml = clusterXmlString;
    if (clusterXml == null) {
        logger.debug("Fetching cluster.xml ...");
        clusterXml = bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY, bootstrapUrls);
    }
    this.cluster = clusterMapper.readCluster(new StringReader(clusterXml), false);
    String storesXml = customStoresXml;
    if (storesXml == null) {
        String storesKey = storeName;
        if (config.isFetchAllStoresXmlInBootstrap()) {
            storesKey = MetadataStore.STORES_KEY;
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Fetching store definition for Store " + storeName + " key " + storesKey);
        }
        storesXml = bootstrapMetadataWithRetries(storesKey, bootstrapUrls);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Obtained cluster metadata xml" + clusterXml);
        logger.debug("Obtained stores  metadata xml" + storesXml);
    }
    storeDefs = storeMapper.readStoreList(new StringReader(storesXml), false);
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null) {
        logger.error("Bootstrap - unknown store: " + storeName);
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug(this.cluster.toString(true));
        logger.debug(storeDef.toString());
    }
    boolean repairReads = !storeDef.isView();
    // construct mapping
    Map<Integer, Store<ByteArray, byte[], byte[]>> clientMapping = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = null;
    if (storeDef.hasHintedHandoffStrategyType())
        slopStores = Maps.newHashMap();
    for (Node node : this.cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> store = getStore(storeDef.getName(), node.getHost(), getPort(node), this.requestFormatType);
        clientMapping.put(node.getId(), store);
        NonblockingStore nonblockingStore = routedStoreFactory.toNonblockingStore(store);
        nonblockingStores.put(node.getId(), nonblockingStore);
        if (slopStores != null) {
            Store<ByteArray, byte[], byte[]> rawSlopStore = getStore("slop", node.getHost(), getPort(node), this.requestFormatType);
            Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(rawSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
            slopStores.put(node.getId(), slopStore);
            nonblockingSlopStores.put(node.getId(), routedStoreFactory.toNonblockingStore(rawSlopStore));
        }
    }
    /*
         * Check if we need to retrieve a reference to the failure detector. For
         * system stores - the FD reference would be passed in.
         */
    FailureDetector failureDetectorRef = fd;
    if (failureDetectorRef == null) {
        failureDetectorRef = getFailureDetector();
    } else {
        logger.debug("Using existing failure detector.");
    }
    this.routedStoreConfig.setRepairReads(repairReads);
    Store<ByteArray, byte[], byte[]> store = routedStoreFactory.create(this.cluster, storeDef, clientMapping, nonblockingStores, slopStores, nonblockingSlopStores, failureDetectorRef, this.routedStoreConfig);
    store = new LoggingStore(store);
    if (isJmxEnabled) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.aggregateStats, this.cachedStoreStats);
        statStore.getStats().registerJmx(identifierString);
        store = statStore;
    }
    if (this.config.isEnableCompressionLayer()) {
        if (storeDef.getKeySerializer().hasCompression() || storeDef.getValueSerializer().hasCompression()) {
            store = new CompressingStore(store, getCompressionStrategy(storeDef.getKeySerializer()), getCompressionStrategy(storeDef.getValueSerializer()));
        }
    }
    /*
         * Initialize the finalstore object only once the store object itself is
         * wrapped by a StatrackingStore seems like the finalstore object is
         * redundant?
         */
    Store<K, V, T> finalStore = (Store<K, V, T>) store;
    if (this.config.isEnableSerializationLayer()) {
        Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(storeDef.getKeySerializer());
        Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(storeDef.getValueSerializer());
        if (storeDef.isView() && (storeDef.getTransformsSerializer() == null))
            throw new SerializationException("Transforms serializer must be specified with a view ");
        Serializer<T> transformsSerializer = (Serializer<T>) serializerFactory.getSerializer(storeDef.getTransformsSerializer() != null ? storeDef.getTransformsSerializer() : new SerializerDefinition("identity"));
        finalStore = SerializingStore.wrap(store, keySerializer, valueSerializer, transformsSerializer);
    }
    // resolver (if they gave us one)
    if (this.config.isEnableInconsistencyResolvingLayer()) {
        InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver() : resolver;
        finalStore = new InconsistencyResolvingStore<K, V, T>(finalStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    }
    return finalStore;
}

Example 2 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testGetAllWithMorePreferredReadsThanNodes.

/**
     * One node up, two preferred reads and one required read. See:
     * 
     * http://github.com/voldemort/voldemort/issues#issue/18
     */
@Test
public void testGetAllWithMorePreferredReadsThanNodes() throws Exception {
    cluster = VoldemortTestConstants.getTwoNodeCluster();
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("test", 2, 2, 1, 2, 2, RoutingStrategyType.CONSISTENT_STRATEGY);
    Map<Integer, Store<ByteArray, byte[], byte[]>> subStores = Maps.newHashMap();
    int id1 = Iterables.get(cluster.getNodes(), 0).getId();
    int id2 = Iterables.get(cluster.getNodes(), 1).getId();
    subStores.put(id1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    subStores.put(id2, new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
    setFailureDetector(subStores);
    routedStoreThreadPool = Executors.newFixedThreadPool(1);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(cluster, storeDef, subStores, failureDetector, createConfig(BANNAGE_PERIOD));
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    store.put(aKey, Versioned.value(aValue), aTransform);
    recordException(failureDetector, cluster.getNodes().iterator().next());
    Map<ByteArray, List<Versioned<byte[]>>> all = store.getAll(Arrays.asList(aKey), Collections.singletonMap(aKey, aTransform));
    assertEquals(1, all.size());
    assertTrue(Arrays.equals(aValue, all.values().iterator().next().get(0).getValue()));
}

Example 3 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testReadRepairWithFailures.

/**
     * See Issue #89: Sequential retrieval in RoutedStore.get doesn't consider
     * repairReads.
     */
@Test
public void testReadRepairWithFailures() throws Exception {
    cluster = getNineNodeCluster();
    RoutedStore routedStore = getStore(cluster, 2, 2, 1, 0);
    BaseStoreRoutingPlan routingPlan = new BaseStoreRoutingPlan(cluster, this.storeDef);
    List<Integer> replicatingNodes = routingPlan.getReplicationNodeList(aKey.get());
    // This is node 1
    Node primaryNode = Iterables.get(cluster.getNodes(), replicatingNodes.get(0));
    // This is node 6
    Node secondaryNode = Iterables.get(cluster.getNodes(), replicatingNodes.get(1));
    // Disable primary node so that the first put happens with 6 as the
    // pseudo master
    recordException(failureDetector, primaryNode);
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    store.put(aKey, new Versioned<byte[]>(aValue), null);
    byte[] anotherValue = "john".getBytes();
    /*
         * Disable the secondary node and enable primary node to prevent the
         * secondary from getting the new version
         */
    recordException(failureDetector, secondaryNode);
    recordSuccess(failureDetector, primaryNode);
    // Generate the clock based off secondary so that the resulting clock
    // will be [1:1, 6:1] across the replicas, except for the secondary
    // which will be [6:1]
    VectorClock clock = getClock(6);
    store.put(aKey, new Versioned<byte[]>(anotherValue, clock), null);
    // Enable secondary and disable primary, the following get should cause
    // a read repair on the secondary in the code path that is only executed
    // if there are failures. This should repair the secondary with the
    // superceding clock [1:1,6:1]
    recordException(failureDetector, primaryNode);
    recordSuccess(failureDetector, secondaryNode);
    List<Versioned<byte[]>> versioneds = store.get(aKey, null);
    assertEquals(1, versioneds.size());
    assertEquals(new ByteArray(anotherValue), new ByteArray(versioneds.get(0).getValue()));
    // Read repairs are done asynchronously, so we sleep for a short period.
    // It may be a good idea to use a synchronous executor service.
    Thread.sleep(500);
    for (Map.Entry<Integer, Store<ByteArray, byte[], byte[]>> innerStoreEntry : routedStore.getInnerStores().entrySet()) {
        // Only look at the nodes in the pref list
        if (replicatingNodes.contains(innerStoreEntry.getKey())) {
            List<Versioned<byte[]>> innerVersioneds = innerStoreEntry.getValue().get(aKey, null);
            assertEquals(1, versioneds.size());
            assertEquals(new ByteArray(anotherValue), new ByteArray(innerVersioneds.get(0).getValue()));
        }
    }
}

Example 4 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testPutWithOneNodeDownAndOneNodeSlowZZZ.

/**
     * See issue #134: RoutedStore put() doesn't wait for enough attempts to
     * succeed
     * 
     * This issue would only happen with one node down and another that was slow
     * to respond in a 3 zone cluster.
     */
@Test
public void testPutWithOneNodeDownAndOneNodeSlowZZZ() throws Exception {
    cluster = VoldemortTestConstants.getSixNodeClusterWith3Zones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(1, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(2, cluster.getNumberOfNodesInZone(0));
    // The replication set for aKey is [1, 2, 0, 3, 5, 4]
    // As per problem statement, set node 2 as the failing node and node 0
    // as the sleepy node
    // PR = RR = 4
    // PW = RW = 4
    // Zone Reads = 0
    // Zone Writes = 0
    // Failing nodes = Node 2
    // Sleepy node = Node 0
    // Threads = 4
    RoutedStore routedStore = getStore(cluster, 4, 4, 0, 0, 4, Sets.newHashSet(2), Sets.newHashSet(0), zoneReplicationFactor, RoutingStrategyType.ZONE_STRATEGY, 0, BANNAGE_PERIOD, new VoldemortException());
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    try {
        store.put(aKey, new Versioned<byte[]>(aValue), aTransform);
    } catch (VoldemortException ve) {
        fail("Unknown exception occurred : " + ve);
    }
}

Example 5 with InconsistencyResolvingStore

use of voldemort.store.versioned.InconsistencyResolvingStore in project voldemort by voldemort.

the class RoutedStoreTest method testBasicOperations.

private void testBasicOperations(int reads, int writes, int failures, int threads, RoutedStore customRoutedStore, long customSleepTime) throws Exception {
    RoutedStore routedStore = null;
    if (customRoutedStore == null) {
        routedStore = getStore(cluster, reads, writes, threads, failures);
    } else {
        routedStore = customRoutedStore;
    }
    Store<ByteArray, byte[], byte[]> store = new InconsistencyResolvingStore<ByteArray, byte[], byte[]>(routedStore, new VectorClockInconsistencyResolver<byte[]>());
    VectorClock clock = getClock(1);
    Versioned<byte[]> versioned = new Versioned<byte[]>(aValue, clock);
    routedStore.put(aKey, versioned, aTransform);
    waitForOperationToComplete(customSleepTime);
    assertNOrMoreEqual(routedStore, cluster.getNumberOfNodes() - failures, aKey, versioned);
    List<Versioned<byte[]>> found = store.get(aKey, aTransform);
    assertEquals(1, found.size());
    assertEquals(versioned, found.get(0));
    waitForOperationToComplete(customSleepTime);
    assertNOrMoreEqual(routedStore, cluster.getNumberOfNodes() - failures, aKey, versioned);
    assertTrue(routedStore.delete(aKey, versioned.getVersion()));
    waitForOperationToComplete(customSleepTime);
    assertNEqual(routedStore, 0, aKey, versioned);
    assertTrue(!routedStore.delete(aKey, versioned.getVersion()));
}