Examples with EventThrottler voldemort.utils.EventThrottler used on opensource projects

Example 6 with EventThrottler

use of voldemort.utils.EventThrottler in project voldemort by voldemort.

the class DataCleanupJob method run.

@Override
public void run() {
    // if the server is neither normal nor offline , skip this run.
    if (!isServerInNormalState() && !isServerInOfflineState()) {
        return;
    }
    StoreDefinition storeDef = null;
    try {
        storeDef = MetadataStore.getStoreDef(storeName, metadataStore);
    } catch (VoldemortException ex) {
        logger.info("Error retrieving store " + storeName + " for data cleanup job ", ex);
        return;
    }
    Integer retentionDays = storeDef.getRetentionDays();
    if (retentionDays == null || retentionDays <= 0) {
        logger.info("Store " + storeName + " does not have retention period set, skipping cleanup job . RetentionDays " + retentionDays);
        return;
    }
    long maxAgeMs = retentionDays * Time.MS_PER_DAY;
    logger.info("Store " + storeName + " cleanup job is starting with RetentionDays " + retentionDays);
    acquireCleanupPermit(scanProgressThisRun, deleteProgressThisRun);
    ClosableIterator<Pair<K, Versioned<V>>> iterator = null;
    try {
        int maxReadRate = storeDef.hasRetentionScanThrottleRate() ? storeDef.getRetentionScanThrottleRate() : Integer.MAX_VALUE;
        EventThrottler throttler = new EventThrottler(maxReadRate);
        store.beginBatchModifications();
        logger.info("Starting data cleanup on store \"" + store.getName() + "\"...");
        long now = time.getMilliseconds();
        iterator = store.entries();
        while (iterator.hasNext()) {
            // check if we have been interrupted
            if (Thread.currentThread().isInterrupted()) {
                logger.info("Datacleanup job halted.");
                return;
            }
            final long INETERVAL = 10000;
            long entriesScanned = scanProgressThisRun.get();
            if (entriesScanned % INETERVAL == 0) {
                if (!isServerInNormalState() && !isServerInOfflineState()) {
                    return;
                }
            }
            scanProgressThisRun.incrementAndGet();
            Pair<K, Versioned<V>> keyAndVal = iterator.next();
            VectorClock clock = (VectorClock) keyAndVal.getSecond().getVersion();
            if (now - clock.getTimestamp() > maxAgeMs) {
                store.delete(keyAndVal.getFirst(), clock);
                final long entriesDeleted = this.deleteProgressThisRun.incrementAndGet();
                if (logger.isDebugEnabled() && entriesDeleted % INETERVAL == 0) {
                    logger.debug("Deleted item " + this.deleteProgressThisRun.get());
                }
            }
            // throttle on number of entries.
            throttler.maybeThrottle(1);
        }
        // log the total items scanned, so we will get an idea of data
        // growth in a cheap, periodic way
        logger.info("Data cleanup on store \"" + store.getName() + "\" is complete; " + this.deleteProgressThisRun.get() + " items deleted. " + scanProgressThisRun.get() + " items scanned");
    } catch (Exception e) {
        logger.error("Error in data cleanup job for store " + store.getName() + ": ", e);
    } finally {
        closeIterator(iterator);
        logger.info("Releasing lock  after data cleanup on \"" + store.getName() + "\".");
        this.cleanupPermits.release(this.getClass().getCanonicalName());
        synchronized (this) {
            totalEntriesScanned += scanProgressThisRun.get();
            scanProgressThisRun.set(0);
            totalEntriesDeleted += deleteProgressThisRun.get();
            deleteProgressThisRun.set(0);
        }
        store.endBatchModifications();
    }
}