Examples with RateLimitingInputStream org.elasticsearch.index.snapshots.blobstore.RateLimitingInputStream used on opensource projects

Example 1 with RateLimitingInputStream

use of org.elasticsearch.index.snapshots.blobstore.RateLimitingInputStream in project crate by crate.

the class BlobStoreRepository method snapshotFile.

/**
 * Snapshot individual file
 * @param fileInfo file to be snapshotted
 */
private void snapshotFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, IndexId indexId, ShardId shardId, SnapshotId snapshotId, IndexShardSnapshotStatus snapshotStatus, Store store) throws IOException {
    final BlobContainer shardContainer = shardContainer(indexId, shardId);
    final String file = fileInfo.physicalName();
    store.incRef();
    try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
        for (int i = 0; i < fileInfo.numberOfParts(); i++) {
            final long partBytes = fileInfo.partBytes(i);
            InputStream inputStream = new InputStreamIndexInput(indexInput, partBytes);
            if (snapshotRateLimiter != null) {
                inputStream = new RateLimitingInputStream(inputStream, snapshotRateLimiter, snapshotRateLimitingTimeInNanos::inc);
            }
            // Make reads abortable by mutating the snapshotStatus object
            inputStream = new FilterInputStream(inputStream) {

                @Override
                public int read() throws IOException {
                    checkAborted();
                    return super.read();
                }

                @Override
                public int read(byte[] b, int off, int len) throws IOException {
                    checkAborted();
                    return super.read(b, off, len);
                }

                private void checkAborted() {
                    if (snapshotStatus.isAborted()) {
                        LOGGER.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileInfo.physicalName());
                        throw new IndexShardSnapshotFailedException(shardId, "Aborted");
                    }
                }
            };
            shardContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes, true);
        }
        Store.verify(indexInput);
        snapshotStatus.addProcessedFile(fileInfo.length());
    } catch (Exception t) {
        failStoreIfCorrupted(store, t);
        snapshotStatus.addProcessedFile(0);
        throw t;
    } finally {
        store.decRef();
    }
}