Examples with ScanPermitWrapper voldemort.server.storage.ScanPermitWrapper used on opensource projects

Example 6 with ScanPermitWrapper

use of voldemort.server.storage.ScanPermitWrapper in project voldemort by voldemort.

the class DataCleanupJobTest method testCleanupFrequency.

@Test
public void testCleanupFrequency() throws InterruptedException {
    SchedulerService scheduler = new SchedulerService(1, time);
    String cleanUpJobName = "cleanup-freq-test";
    try {
        MockTime mockTime = new MockTime(System.currentTimeMillis());
        // clean up will purge everything older than last 2 seconds
        DataCleanupJob cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine, new ScanPermitWrapper(1), STORE_NAME, mockTime, metadataStore);
        // and will run every 3 seconds starting now
        scheduler.schedule(cleanUpJobName, cleanupJob, new Date(), 1 * Time.MS_PER_SECOND);
        // load some data
        putWithTimeStamp(0, 10, System.currentTimeMillis());
        Thread.sleep(3 * Time.MS_PER_SECOND);
        // None of the keys should have been deleted
        assertPresence(0, 10);
        assertEquals("No entries must be deleted now", 0, cleanupJob.getEntriesDeleted());
        assertTrue("Entries are scanned, but not deleted", cleanupJob.getEntriesScanned() > 0);
        // Increase the mockTime by a day and a millisecond.
        mockTime.setTime(System.currentTimeMillis() + START_RETENTION * Time.MS_PER_DAY + 1);
        // load some more data
        putWithTimeStamp(10, 20, mockTime.getMilliseconds());
        // give time for data cleanup to run
        Thread.sleep(3 * Time.MS_PER_SECOND);
        // first batch of writes should have been deleted
        assertAbsence(0, 10);
        // and later ones retained.
        assertPresence(10, 20);
        assertEquals("10 entries should be deleted", 10, cleanupJob.getEntriesDeleted());
        // Insert 10 keys that are separated by a day and millisecond
        long currentTime = mockTime.getMilliseconds();
        for (int i = 20; i < 30; i++) {
            currentTime += Time.MS_PER_DAY + 1;
            putWithTimeStamp(i, i + 1, currentTime);
        }
        mockTime.setTime(currentTime);
        updateStoreDef(REDUCED_RETENTION);
        Thread.sleep(3 * Time.MS_PER_SECOND);
        assertAbsence(10, 28);
        assertEquals("28 entries should be deleted", 28, cleanupJob.getEntriesDeleted());
        // Only last 2 keys should be present, third key is old by 2 days
        // and 2 milliseconds
        assertPresence(28, 30);
    } finally {
        scheduler.terminate(cleanUpJobName);
        scheduler.stop();
    }
}

Example 7 with ScanPermitWrapper

use of voldemort.server.storage.ScanPermitWrapper in project voldemort by voldemort.

the class StreamingSlopPusherTest method testFailedServer.

@Test
public void testFailedServer() throws IOException, InterruptedException {
    startServers(0, 2);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    // Generate slops for 2
    final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        // did it get deleted correctly
        assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 50);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 2);
}

Example 8 with ScanPermitWrapper

use of voldemort.server.storage.ScanPermitWrapper in project voldemort by voldemort.

the class StreamingSlopPusherTest method testOutOfOrder.

/**
     * Tests that everything works even if the slops get replayed out of order
     */
@Test
@Ignore
public void testOutOfOrder() throws InterruptedException, IOException {
    startServers(0, 1);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // 5 slops for 3 stores => ( key1 [put(value1), delete, put(value2)] AND
    // key2 [put(value2), delete] )
    long keyInt1 = (long) (Math.random() * 1000000000L), keyInt2 = (long) (Math.random() * 10000000L);
    ByteArray key1 = new ByteArray(ByteUtils.getBytes("" + keyInt1, "UTF-8")), key2 = new ByteArray(ByteUtils.getBytes("" + keyInt2, "UTF-8"));
    byte[] value1 = ByteUtils.getBytes("value-" + new String(key1.get()), "UTF-8"), value2 = ByteUtils.getBytes("value-" + new String(key2.get()), "UTF-8");
    VectorClock vectorClock1 = new VectorClock(), vectorClock2 = new VectorClock();
    List<Versioned<Slop>> entrySet = Lists.newArrayList();
    for (String storeName : Lists.newArrayList("test-replication-memory", "users", "test-replication-persistent")) {
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key1, value1, null, 1, new Date()), vectorClock1));
        vectorClock1 = vectorClock1.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.DELETE, key1, null, null, 1, new Date()), vectorClock1));
        vectorClock1 = vectorClock1.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key1, value2, null, 1, new Date()), vectorClock1));
        vectorClock2 = vectorClock2.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.PUT, key2, value2, null, 1, new Date()), vectorClock2));
        vectorClock2 = vectorClock2.incremented(0, System.currentTimeMillis());
        entrySet.add(Versioned.value(new Slop(storeName, Slop.Operation.DELETE, key2, null, null, 1, new Date()), vectorClock2));
    }
    Collections.shuffle(entrySet);
    // Generate two sub lists to send in batches
    List<Versioned<Slop>> firstSet = entrySet.subList(0, 7), secondSet = entrySet.subList(7, 15);
    populateSlops(0, slopStoreNode0, firstSet);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    populateSlops(0, slopStoreNode0, secondSet);
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    for (String storeName : Lists.newArrayList("test-replication-memory", "users", "test-replication-persistent")) {
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(storeName);
        assertEquals(store.get(key1, null).size(), 1);
        assertEquals(ByteUtils.compare(store.get(key1, null).get(0).getValue(), value2), 0);
        assertEquals(store.get(key2, null).size(), 0);
    }
    stopServers(0, 1);
}

Example 9 with ScanPermitWrapper

use of voldemort.server.storage.ScanPermitWrapper in project voldemort by voldemort.

the class StreamingSlopPusherTest method testNormalPush.

@Test
public void testNormalPush() throws InterruptedException, IOException {
    startServers(0, 1);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet = ServerTestUtils.createRandomSlops(1, 100, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 0);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(0));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 1);
}