Examples with CLEANABLE_SHARDS_BATCH_SIZE com.facebook.presto.raptor.metadata.ShardDao.CLEANABLE_SHARDS_BATCH_SIZE used on opensource projects

Example 1 with CLEANABLE_SHARDS_BATCH_SIZE

use of com.facebook.presto.raptor.metadata.ShardDao.CLEANABLE_SHARDS_BATCH_SIZE in project presto by prestodb.

the class ShardCleaner method cleanBackupShards.

@VisibleForTesting
void cleanBackupShards() {
    Set<UUID> processing = newConcurrentHashSet();
    BlockingQueue<UUID> completed = new LinkedBlockingQueue<>();
    boolean fill = true;
    while (!Thread.currentThread().isInterrupted()) {
        // get a new batch if any completed and we are under the batch size
        Set<UUID> uuids = ImmutableSet.of();
        if (fill && (processing.size() < CLEANABLE_SHARDS_BATCH_SIZE)) {
            uuids = dao.getCleanableShardsBatch(maxTimestamp(backupCleanTime));
            fill = false;
        }
        if (uuids.isEmpty() && processing.isEmpty()) {
            break;
        }
        // skip any that are already processing and mark remaining as processing
        uuids = ImmutableSet.copyOf(difference(uuids, processing));
        processing.addAll(uuids);
        // execute deletes
        for (UUID uuid : uuids) {
            runAsync(() -> backupStore.get().deleteShard(uuid), backupExecutor).thenAccept(v -> completed.add(uuid)).whenComplete((v, e) -> {
                if (e != null) {
                    log.error(e, "Error cleaning backup shard: %s", uuid);
                    backupJobErrors.update(1);
                    processing.remove(uuid);
                }
            });
        }
        // get the next batch of completed deletes
        int desired = min(100, processing.size());
        Collection<UUID> done = drain(completed, desired, 100, MILLISECONDS);
        if (done.isEmpty()) {
            continue;
        }
        // remove completed deletes from database
        processing.removeAll(done);
        dao.deleteCleanedShards(done);
        backupShardsCleaned.update(done.size());
        fill = true;
    }
}