Examples with FailureDetectorConfig voldemort.cluster.failuredetector.FailureDetectorConfig used on opensource projects

Example 11 with FailureDetectorConfig

use of voldemort.cluster.failuredetector.FailureDetectorConfig in project voldemort by voldemort.

the class RoutedStoreTest method setFailureDetector.

/**
     * Function to set the failure detector class.
     * 
     * Note: We set this to BannagePeriodFailureDetector which is not supported
     * or recommended anymore because of the inefficiency in checking for
     * liveness and the potential issue of marking all the nodes as unavailable.
     * However, for the purpose of these tests, BannagePeriodFailureDetector is
     * useful to deterministically set a node as available or unavailable.
     * 
     * @param subStores Stores used to check if node is up
     * @throws Exception
     */
private void setFailureDetector(Map<Integer, Store<ByteArray, byte[], byte[]>> subStores) throws Exception {
    // (the final one is destroyed in tearDown).
    if (failureDetector != null)
        failureDetector.destroy();
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setBannagePeriod(BANNAGE_PERIOD).setCluster(cluster).setConnectionVerifier(create(subStores));
    failureDetector = create(failureDetectorConfig, false);
}

Example 12 with FailureDetectorConfig

use of voldemort.cluster.failuredetector.FailureDetectorConfig in project voldemort by voldemort.

the class StreamingSlopPusherTest method testFailedServer.

@Test
public void testFailedServer() throws IOException, InterruptedException {
    startServers(0, 2);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    // Generate slops for 2
    final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        // did it get deleted correctly
        assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 50);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 2);
}

Example 13 with FailureDetectorConfig

use of voldemort.cluster.failuredetector.FailureDetectorConfig in project voldemort by voldemort.

the class StreamingSlopPusherTest method testOutOfOrder.

/**
     * Tests that everything works even if the slops get replayed out of order
     */
@Test
@Ignore
public void testOutOfOrder() throws InterruptedException, IOException {
    startServers(0, 1);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // 5 slops for 3 stores => ( key1 [put(value1), delete, put(value2)] AND
    // key2 [put(value2), delete] )
    long keyInt1 = (long) (Math.random() * 1000000000L), keyInt2 = (long) (Math.random() * 10000000L);
    ByteArray key1 = new ByteArray(ByteUtils.getBytes("" + keyInt1, "UTF-8")), key2 = new ByteArray(ByteUtils.getBytes("" + keyInt2, "UTF-8"));
    byte[] value1 = ByteUtils.getBytes("value-" + new String(key1.get()), "UTF-8"), value2 = ByteUtils.getBytes("value-" + new String(key2.get()), "UTF-8");
    VectorClock vectorClock1 = new VectorClock(), vectorClock2 = new VectorClock();
    List<Versioned<Slop>> entrySet = Lists.newArrayList();
    for (String storeName : Lists.newArrayList("test-replication-memory", "users", "test-replication-persistent")) {
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key1, value1, null, 1, new Date()), vectorClock1));
        vectorClock1 = vectorClock1.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.DELETE, key1, null, null, 1, new Date()), vectorClock1));
        vectorClock1 = vectorClock1.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key1, value2, null, 1, new Date()), vectorClock1));
        vectorClock2 = vectorClock2.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key2, value2, null, 1, new Date()), vectorClock2));
        vectorClock2 = vectorClock2.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.DELETE, key2, null, null, 1, new Date()), vectorClock2));
    }
    Collections.shuffle(entrySet);
    // Generate two sub lists to send in batches
    List<Versioned<Slop>> firstSet = entrySet.subList(0, 7), secondSet = entrySet.subList(7, 15);
    populateSlops(0, slopStoreNode0, firstSet);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    populateSlops(0, slopStoreNode0, secondSet);
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    for (String storeName : Lists.newArrayList("test-replication-memory", "users", "test-replication-persistent")) {
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(storeName);
        assertEquals(store.get(key1, null).size(), 1);
        assertEquals(ByteUtils.compare(store.get(key1, null).get(0).getValue(), value2), 0);
        assertEquals(store.get(key2, null).size(), 0);
    }
    stopServers(0, 1);
}

Example 14 with FailureDetectorConfig

use of voldemort.cluster.failuredetector.FailureDetectorConfig in project voldemort by voldemort.

the class StreamingSlopPusherTest method testNormalPush.

@Test
public void testNormalPush() throws InterruptedException, IOException {
    startServers(0, 1);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet = ServerTestUtils.createRandomSlops(1, 100, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 0);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(0));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 1);
}

Example 15 with FailureDetectorConfig

use of voldemort.cluster.failuredetector.FailureDetectorConfig in project voldemort by voldemort.

the class ClusterTest method setUp.

@Override
@Before
public void setUp() throws Exception {
    this.nodes = ImmutableList.of(new Node(1, "test1", 1, 1, 1, ImmutableList.of(0, 1, 2)), new Node(2, "test1", 2, 2, 2, ImmutableList.of(3, 4, 5)), new Node(3, "test1", 3, 3, 3, ImmutableList.of(6, 7, 8)), new Node(4, "test1", 4, 4, 4, ImmutableList.of(9, 10, 11)));
    this.cluster = new Cluster(clusterName, nodes);
    this.time = SystemTime.INSTANCE;
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(failureDetectorClass.getName()).setBannagePeriod(1000).setCluster(cluster).setConnectionVerifier(create(cluster.getNodes())).setTime(time);
    failureDetector = create(failureDetectorConfig, false);
}