Examples with InputStreamWithMeta org.apache.storm.blobstore.InputStreamWithMeta used on opensource projects

Example 6 with InputStreamWithMeta

use of org.apache.storm.blobstore.InputStreamWithMeta in project storm by apache.

the class LocallyCachedBlob method fetch.

/**
 * Helper function to download blob from blob store.
 * @param store Blob store to fetch blobs from
 * @param key Key to retrieve blobs
 * @param pathSupplier A function that supplies the download destination of a blob. It guarantees the validity
 *                     of path or throws {@link IOException}
 * @param outStreamSupplier A function that supplies the {@link OutputStream} object
 * @return The metadata of the download session, including blob's version and download destination
 * @throws KeyNotFoundException Thrown if key to retrieve blob is invalid
 * @throws AuthorizationException Thrown if the retrieval is not under security authorization
 * @throws IOException Thrown if any IO error occurs
 */
protected DownloadMeta fetch(ClientBlobStore store, String key, IOFunction<Long, Path> pathSupplier, IOFunction<File, OutputStream> outStreamSupplier) throws KeyNotFoundException, AuthorizationException, IOException {
    try (InputStreamWithMeta in = store.getBlob(key)) {
        long newVersion = in.getVersion();
        long currentVersion = getLocalVersion();
        if (newVersion == currentVersion) {
            LOG.warn("The version did not change, but going to download again {} {}", currentVersion, key);
        }
        // Make sure the parent directory is there and ready to go
        Path downloadPath = pathSupplier.apply(newVersion);
        LOG.debug("Downloading {} to {}", key, downloadPath);
        long duration;
        long totalRead = 0;
        try (OutputStream out = outStreamSupplier.apply(downloadPath.toFile())) {
            long startTime = Time.nanoTime();
            byte[] buffer = new byte[4096];
            int read;
            while ((read = in.read(buffer)) >= 0) {
                out.write(buffer, 0, read);
                totalRead += read;
            }
            duration = Time.nanoTime() - startTime;
        }
        long expectedSize = in.getFileLength();
        if (totalRead != expectedSize) {
            throw new IOException("We expected to download " + expectedSize + " bytes but found we got " + totalRead);
        } else {
            double downloadRate = ((double) totalRead * 1e3) / duration;
            fetchingRate.update(Math.round(downloadRate));
        }
        return new DownloadMeta(downloadPath, newVersion);
    }
}