Examples with VisibleForTesting com.google.common.annotations.VisibleForTesting used on opensource projects

Example 56 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project presto by prestodb.

the class OrcStorageManager method rewriteShard.

@VisibleForTesting
Collection<Slice> rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    UUID newShardUuid = UUID.randomUUID();
    File input = storageService.getStorageFile(shardUuid);
    File output = storageService.getStagingFile(newShardUuid);
    OrcFileInfo info = rewriteFile(input, output, rowsToDelete);
    long rowCount = info.getRowCount();
    if (rowCount == 0) {
        return shardDelta(shardUuid, Optional.empty());
    }
    shardRecorder.recordCreatedShard(transactionId, newShardUuid);
    // submit for backup and wait until it finishes
    getFutureValue(backupManager.submit(newShardUuid, output));
    Set<String> nodes = ImmutableSet.of(nodeId);
    long uncompressedSize = info.getUncompressedSize();
    ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize);
    writeShard(newShardUuid);
    return shardDelta(shardUuid, Optional.of(shard));
}

Example 57 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project presto by prestodb.

the class OrcStorageManager method openShard.

@VisibleForTesting
OrcDataSource openShard(UUID shardUuid, ReaderAttributes readerAttributes) {
    File file = storageService.getStorageFile(shardUuid).getAbsoluteFile();
    if (!file.exists() && backupStore.isPresent()) {
        try {
            Future<?> future = recoveryManager.recoverShard(shardUuid);
            future.get(recoveryTimeout.toMillis(), TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw Throwables.propagate(e);
        } catch (ExecutionException e) {
            propagateIfInstanceOf(e.getCause(), PrestoException.class);
            throw new PrestoException(RAPTOR_RECOVERY_ERROR, "Error recovering shard " + shardUuid, e.getCause());
        } catch (TimeoutException e) {
            throw new PrestoException(RAPTOR_RECOVERY_TIMEOUT, "Shard is being recovered from backup. Please retry in a few minutes: " + shardUuid);
        }
    }
    try {
        return fileOrcDataSource(readerAttributes, file);
    } catch (IOException e) {
        throw new PrestoException(RAPTOR_ERROR, "Failed to open shard file: " + file, e);
    }
}

Example 58 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project presto by prestodb.

the class ShardCleaner method cleanBackupShards.

@VisibleForTesting
void cleanBackupShards() {
    Set<UUID> processing = newConcurrentHashSet();
    BlockingQueue<UUID> completed = new LinkedBlockingQueue<>();
    boolean fill = true;
    while (!Thread.currentThread().isInterrupted()) {
        // get a new batch if any completed and we are under the batch size
        Set<UUID> uuids = ImmutableSet.of();
        if (fill && (processing.size() < CLEANABLE_SHARDS_BATCH_SIZE)) {
            uuids = dao.getCleanableShardsBatch(maxTimestamp(backupCleanTime));
            fill = false;
        }
        if (uuids.isEmpty() && processing.isEmpty()) {
            break;
        }
        // skip any that are already processing and mark remaining as processing
        uuids = ImmutableSet.copyOf(difference(uuids, processing));
        processing.addAll(uuids);
        // execute deletes
        for (UUID uuid : uuids) {
            runAsync(() -> backupStore.get().deleteShard(uuid), backupExecutor).thenAccept(v -> completed.add(uuid)).whenComplete((v, e) -> {
                if (e != null) {
                    log.error(e, "Error cleaning backup shard: %s", uuid);
                    backupJobErrors.update(1);
                    processing.remove(uuid);
                }
            });
        }
        // get the next batch of completed deletes
        int desired = min(100, processing.size());
        Collection<UUID> done = drain(completed, desired, 100, MILLISECONDS);
        if (done.isEmpty()) {
            continue;
        }
        // remove completed deletes from database
        processing.removeAll(done);
        dao.deleteCleanedShards(done);
        backupShardsCleaned.update(done.size());
        fill = true;
    }
}

Example 59 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project presto by prestodb.

the class ShardCleaner method cleanLocalShards.

@VisibleForTesting
synchronized void cleanLocalShards() {
    // find all files on the local node
    Set<UUID> local = storageService.getStorageShards();
    // get shards assigned to the local node
    Set<UUID> assigned = dao.getNodeShards(currentNode, null).stream().map(ShardMetadata::getShardUuid).collect(toSet());
    // un-mark previously marked files that are now assigned
    for (UUID uuid : assigned) {
        shardsToClean.remove(uuid);
    }
    // mark all files that are not assigned
    for (UUID uuid : local) {
        if (!assigned.contains(uuid)) {
            shardsToClean.putIfAbsent(uuid, ticker.read());
        }
    }
    // delete files marked earlier than the clean interval
    long threshold = ticker.read() - localCleanTime.roundTo(NANOSECONDS);
    Set<UUID> deletions = shardsToClean.entrySet().stream().filter(entry -> entry.getValue() < threshold).map(Map.Entry::getKey).collect(toSet());
    if (deletions.isEmpty()) {
        return;
    }
    for (UUID uuid : deletions) {
        deleteFile(storageService.getStorageFile(uuid));
        shardsToClean.remove(uuid);
    }
    localShardsCleaned.update(deletions.size());
    log.info("Cleaned %s local shards", deletions.size());
}

Example 60 with VisibleForTesting

use of com.google.common.annotations.VisibleForTesting in project presto by prestodb.

the class ShardRecoveryManager method restoreFromBackup.

@VisibleForTesting
void restoreFromBackup(UUID shardUuid, OptionalLong shardSize) {
    File storageFile = storageService.getStorageFile(shardUuid);
    if (!backupStore.get().shardExists(shardUuid)) {
        stats.incrementShardRecoveryBackupNotFound();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid);
    }
    if (storageFile.exists()) {
        if (!shardSize.isPresent() || (storageFile.length() == shardSize.getAsLong())) {
            return;
        }
        log.warn("Local shard file is corrupt. Deleting local file: %s", storageFile);
        storageFile.delete();
    }
    // create a temporary file in the staging directory
    File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid));
    storageService.createParents(stagingFile);
    // copy to temporary file
    log.info("Copying shard %s from backup...", shardUuid);
    long start = System.nanoTime();
    try {
        backupStore.get().restoreShard(shardUuid, stagingFile);
    } catch (PrestoException e) {
        stats.incrementShardRecoveryFailure();
        stagingFile.delete();
        throw e;
    }
    Duration duration = nanosSince(start);
    DataSize size = succinctBytes(stagingFile.length());
    DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize();
    stats.addShardRecoveryDataRate(rate, size, duration);
    log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate);
    // move to final location
    storageService.createParents(storageFile);
    try {
        Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE);
    } catch (FileAlreadyExistsException e) {
    // someone else already created it (should not happen, but safe to ignore)
    } catch (IOException e) {
        stats.incrementShardRecoveryFailure();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e);
    } finally {
        stagingFile.delete();
    }
    if (!storageFile.exists() || (shardSize.isPresent() && (storageFile.length() != shardSize.getAsLong()))) {
        stats.incrementShardRecoveryFailure();
        log.info("Files do not match after recovery. Deleting local file: " + shardUuid);
        storageFile.delete();
        throw new PrestoException(RAPTOR_RECOVERY_ERROR, "File not recovered correctly: " + shardUuid);
    }
    stats.incrementShardRecoverySuccess();
}