Examples with DataCleanupJob voldemort.server.scheduler.DataCleanupJob used on opensource projects

Example 1 with DataCleanupJob

use of voldemort.server.scheduler.DataCleanupJob in project voldemort by voldemort.

the class DataCleanupJobTest method testStoreDeletion.

@Test
public void testStoreDeletion() throws InterruptedException {
    SchedulerService scheduler = new SchedulerService(1, time);
    String cleanUpJobName = "cleanup-freq-test";
    try {
        MockTime mockTime = new MockTime(System.currentTimeMillis());
        ScanPermitWrapper scanWrapper = new ScanPermitWrapper(1);
        // clean up will purge everything older than last 2 seconds
        DataCleanupJob cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine, new ScanPermitWrapper(1), STORE_NAME, mockTime, metadataStore);
        // and will run every 3 seconds starting now
        scheduler.schedule(cleanUpJobName, cleanupJob, new Date(), 1 * Time.MS_PER_SECOND);
        // Insert records that should be deleted when the DataCleanUp Job
        // runs
        putWithTimeStamp(0, 10, System.currentTimeMillis() - 8 * Time.MS_PER_DAY);
        Thread.sleep(3 * Time.MS_PER_SECOND);
        // All of them should be deleted.
        assertAbsence(0, 10);
        // Delete the store.
        metadataStore.deleteStoreDefinition(STORE_NAME);
        // Wait 2 seconds to give the Scheduler job, come out of the
        // previous scan if any
        Thread.sleep(2 * Time.MS_PER_SECOND);
        // Intermittent failure, means problem with the code which needs to be fixed.
        for (int i = 0; i < 1000; i++) {
            assertEquals("Deleted store should never acquire a scan permit", 1, scanWrapper.availablePermits());
        }
    } finally {
        scheduler.terminate(cleanUpJobName);
        scheduler.stop();
    }
}

Example 2 with DataCleanupJob

use of voldemort.server.scheduler.DataCleanupJob in project voldemort by voldemort.

the class StorageService method scheduleCleanupJob.

/**
     * Schedule a data retention cleanup job for the given store
     * 
     * @param storeDef The store definition
     * @param engine The storage engine to do cleanup on
     */
private void scheduleCleanupJob(StoreDefinition storeDef, StorageEngine<ByteArray, byte[], byte[]> engine) {
    // Compute the start time of the job, based on current time
    GregorianCalendar cal = Utils.getCalendarForNextRun(new GregorianCalendar(), voldemortConfig.getRetentionCleanupFirstStartDayOfWeek(), voldemortConfig.getRetentionCleanupFirstStartTimeInHour());
    // allow only one cleanup job at a time
    Date startTime = cal.getTime();
    logger.info("Scheduling data retention cleanup job for store '" + storeDef.getName() + "' at " + startTime);
    Runnable cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine, scanPermitWrapper, storeDef.getName(), SystemTime.INSTANCE, metadata);
    if (voldemortConfig.isJmxEnabled()) {
        JmxUtils.registerMbean("DataCleanupJob-" + engine.getName(), cleanupJob);
    }
    long retentionFreqHours = storeDef.hasRetentionFrequencyDays() ? (storeDef.getRetentionFrequencyDays() * Time.HOURS_PER_DAY) : voldemortConfig.getRetentionCleanupScheduledPeriodInHour();
    this.scheduler.schedule("cleanup-" + storeDef.getName(), cleanupJob, startTime, retentionFreqHours * Time.MS_PER_HOUR, voldemortConfig.getRetentionCleanupPinStartTime());
}

Example 3 with DataCleanupJob

use of voldemort.server.scheduler.DataCleanupJob in project voldemort by voldemort.

the class DataCleanupJobTest method testCleanupFrequency.

@Test
public void testCleanupFrequency() throws InterruptedException {
    SchedulerService scheduler = new SchedulerService(1, time);
    String cleanUpJobName = "cleanup-freq-test";
    try {
        MockTime mockTime = new MockTime(System.currentTimeMillis());
        // clean up will purge everything older than last 2 seconds
        DataCleanupJob cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine, new ScanPermitWrapper(1), STORE_NAME, mockTime, metadataStore);
        // and will run every 3 seconds starting now
        scheduler.schedule(cleanUpJobName, cleanupJob, new Date(), 1 * Time.MS_PER_SECOND);
        // load some data
        putWithTimeStamp(0, 10, System.currentTimeMillis());
        Thread.sleep(3 * Time.MS_PER_SECOND);
        // None of the keys should have been deleted
        assertPresence(0, 10);
        assertEquals("No entries must be deleted now", 0, cleanupJob.getEntriesDeleted());
        assertTrue("Entries are scanned, but not deleted", cleanupJob.getEntriesScanned() > 0);
        // Increase the mockTime by a day and a millisecond.
        mockTime.setTime(System.currentTimeMillis() + START_RETENTION * Time.MS_PER_DAY + 1);
        // load some more data
        putWithTimeStamp(10, 20, mockTime.getMilliseconds());
        // give time for data cleanup to run
        Thread.sleep(3 * Time.MS_PER_SECOND);
        // first batch of writes should have been deleted
        assertAbsence(0, 10);
        // and later ones retained.
        assertPresence(10, 20);
        assertEquals("10 entries should be deleted", 10, cleanupJob.getEntriesDeleted());
        // Insert 10 keys that are separated by a day and millisecond
        long currentTime = mockTime.getMilliseconds();
        for (int i = 20; i < 30; i++) {
            currentTime += Time.MS_PER_DAY + 1;
            putWithTimeStamp(i, i + 1, currentTime);
        }
        mockTime.setTime(currentTime);
        updateStoreDef(REDUCED_RETENTION);
        Thread.sleep(3 * Time.MS_PER_SECOND);
        assertAbsence(10, 28);
        assertEquals("28 entries should be deleted", 28, cleanupJob.getEntriesDeleted());
        // Only last 2 keys should be present, third key is old by 2 days
        // and 2 milliseconds
        assertPresence(28, 30);
    } finally {
        scheduler.terminate(cleanUpJobName);
        scheduler.stop();
    }
}