Examples with BlobRequestOptions com.microsoft.azure.storage.blob.BlobRequestOptions used on opensource projects

Example 1 with BlobRequestOptions

use of com.microsoft.azure.storage.blob.BlobRequestOptions in project hadoop by apache.

the class AzureNativeFileSystemStore method rename.

@Override
public void rename(String srcKey, String dstKey, boolean acquireLease, SelfRenewingLease existingLease) throws IOException {
    LOG.debug("Moving {} to {}", srcKey, dstKey);
    if (acquireLease && existingLease != null) {
        throw new IOException("Cannot acquire new lease if one already exists.");
    }
    CloudBlobWrapper srcBlob = null;
    CloudBlobWrapper dstBlob = null;
    SelfRenewingLease lease = null;
    try {
        // storage server.
        if (null == storageInteractionLayer) {
            final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
            throw new AssertionError(errMsg);
        }
        checkContainer(ContainerAccessType.ReadThenWrite);
        // Get the source blob and assert its existence. If the source key
        // needs to be normalized then normalize it.
        //
        srcBlob = getBlobReference(srcKey);
        if (!srcBlob.exists(getInstrumentedContext())) {
            throw new AzureException("Source blob " + srcKey + " does not exist.");
        }
        /**
       * Conditionally get a lease on the source blob to prevent other writers
       * from changing it. This is used for correctness in HBase when log files
       * are renamed. It generally should do no harm other than take a little
       * more time for other rename scenarios. When the HBase master renames a
       * log file folder, the lease locks out other writers.  This
       * prevents a region server that the master thinks is dead, but is still
       * alive, from committing additional updates.  This is different than
       * when HBase runs on HDFS, where the region server recovers the lease
       * on a log file, to gain exclusive access to it, before it splits it.
       */
        if (acquireLease) {
            lease = srcBlob.acquireLease();
        } else if (existingLease != null) {
            lease = existingLease;
        }
        // Get the destination blob. The destination key always needs to be
        // normalized.
        //
        dstBlob = getBlobReference(dstKey);
        // throttled.
        try {
            dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
        } catch (StorageException se) {
            if (se.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
                int copyBlobMinBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MIN_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL);
                int copyBlobMaxBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL);
                int copyBlobDeltaBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_BACKOFF_INTERVAL);
                int copyBlobMaxRetries = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_IO_RETRIES, DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS);
                BlobRequestOptions options = new BlobRequestOptions();
                options.setRetryPolicyFactory(new RetryExponentialRetry(copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries));
                dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
            } else {
                throw se;
            }
        }
        waitForCopyToComplete(dstBlob, getInstrumentedContext());
        safeDelete(srcBlob, lease);
    } catch (StorageException e) {
        if (e.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
            LOG.warn("Rename: CopyBlob: StorageException: ServerBusy: Retry complete, will attempt client side copy for page blob");
            InputStream ipStream = null;
            OutputStream opStream = null;
            try {
                if (srcBlob.getProperties().getBlobType() == BlobType.PAGE_BLOB) {
                    ipStream = openInputStream(srcBlob);
                    opStream = openOutputStream(dstBlob);
                    byte[] buffer = new byte[PageBlobFormatHelpers.PAGE_SIZE];
                    int len;
                    while ((len = ipStream.read(buffer)) != -1) {
                        opStream.write(buffer, 0, len);
                    }
                    opStream.flush();
                    opStream.close();
                    ipStream.close();
                } else {
                    throw new AzureException(e);
                }
                safeDelete(srcBlob, lease);
            } catch (StorageException se) {
                LOG.warn("Rename: CopyBlob: StorageException: Failed");
                throw new AzureException(se);
            } finally {
                IOUtils.closeStream(ipStream);
                IOUtils.closeStream(opStream);
            }
        } else {
            throw new AzureException(e);
        }
    } catch (URISyntaxException e) {
        // Re-throw exception as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 2 with BlobRequestOptions

use of com.microsoft.azure.storage.blob.BlobRequestOptions in project hadoop by apache.

the class AzureNativeFileSystemStore method getDownloadOptions.

private BlobRequestOptions getDownloadOptions() {
    BlobRequestOptions options = new BlobRequestOptions();
    options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries));
    options.setUseTransactionalContentMD5(getUseTransactionalContentMD5());
    return options;
}

Example 3 with BlobRequestOptions

use of com.microsoft.azure.storage.blob.BlobRequestOptions in project hadoop by apache.

the class BlockBlobAppendStream method setBlocksCountAndBlockIdPrefix.

/**
   * Helper method used to generate the blockIDs. The algorithm used is similar to the Azure
   * storage SDK.
   */
private void setBlocksCountAndBlockIdPrefix() throws IOException {
    try {
        if (nextBlockCount == UNSET_BLOCKS_COUNT && blockIdPrefix == null) {
            List<BlockEntry> blockEntries = blob.downloadBlockList(BlockListingFilter.COMMITTED, new BlobRequestOptions(), opContext);
            String blockZeroBlockId = (blockEntries.size() > 0) ? blockEntries.get(0).getId() : "";
            String prefix = UUID.randomUUID().toString() + "-";
            String sampleNewerVersionBlockId = generateNewerVersionBlockId(prefix, 0);
            if (blockEntries.size() > 0 && blockZeroBlockId.length() < sampleNewerVersionBlockId.length()) {
                // If blob has already been created with 2.2.0, append subsequent blocks with older version (2.2.0) blockId
                // compute nextBlockCount, the way it was done before; and don't use blockIdPrefix
                this.blockIdPrefix = "";
                nextBlockCount = (long) (sequenceGenerator.nextInt(Integer.MAX_VALUE)) + sequenceGenerator.nextInt(Integer.MAX_VALUE - MAX_BLOCK_COUNT);
                nextBlockCount += blockEntries.size();
            } else {
                // If there are no existing blocks, create the first block with newer version (4.2.0) blockId
                // If blob has already been created with 4.2.0, append subsequent blocks with newer version (4.2.0) blockId
                this.blockIdPrefix = prefix;
                nextBlockCount = blockEntries.size();
            }
        }
    } catch (StorageException ex) {
        LOG.debug("Encountered storage exception during setting next Block Count and BlockId prefix." + " StorageException : {} ErrorCode : {}", ex, ex.getErrorCode());
        throw new IOException(ex);
    }
}

Example 4 with BlobRequestOptions

use of com.microsoft.azure.storage.blob.BlobRequestOptions in project hadoop by apache.

the class PageBlobInputStream method getPageBlobDataSize.

/**
   * Helper method to extract the actual data size of a page blob.
   * This typically involves 2 service requests (one for page ranges, another
   * for the last page's data).
   *
   * @param blob The blob to get the size from.
   * @param opContext The operation context to use for the requests.
   * @return The total data size of the blob in bytes.
   * @throws IOException If the format is corrupt.
   * @throws StorageException If anything goes wrong in the requests.
   */
public static long getPageBlobDataSize(CloudPageBlobWrapper blob, OperationContext opContext) throws IOException, StorageException {
    // Get the page ranges for the blob. There should be one range starting
    // at byte 0, but we tolerate (and ignore) ranges after the first one.
    ArrayList<PageRange> pageRanges = blob.downloadPageRanges(new BlobRequestOptions(), opContext);
    if (pageRanges.size() == 0) {
        return 0;
    }
    if (pageRanges.get(0).getStartOffset() != 0) {
        // starting at byte 0.
        throw badStartRangeException(blob, pageRanges.get(0));
    }
    long totalRawBlobSize = pageRanges.get(0).getEndOffset() + 1;
    // Get the last page.
    long lastPageStart = totalRawBlobSize - PAGE_SIZE;
    ByteArrayOutputStream baos = new ByteArrayOutputStream(PageBlobFormatHelpers.PAGE_SIZE);
    blob.downloadRange(lastPageStart, PAGE_SIZE, baos, new BlobRequestOptions(), opContext);
    byte[] lastPage = baos.toByteArray();
    short lastPageSize = getPageSize(blob, lastPage, 0);
    long totalNumberOfPages = totalRawBlobSize / PAGE_SIZE;
    return (totalNumberOfPages - 1) * PAGE_DATA_SIZE + lastPageSize;
}

Example 5 with BlobRequestOptions

use of com.microsoft.azure.storage.blob.BlobRequestOptions in project jackrabbit-oak by apache.

the class AzureBlobStoreBackend method write.

@Override
public void write(DataIdentifier identifier, File file) throws DataStoreException {
    if (null == identifier) {
        throw new NullPointerException("identifier");
    }
    if (null == file) {
        throw new NullPointerException("file");
    }
    String key = getKeyName(identifier);
    long start = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        long len = file.length();
        LOG.debug("Blob write started. identifier={} length={}", key, len);
        CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
        if (!blob.exists()) {
            BlobRequestOptions options = new BlobRequestOptions();
            options.setConcurrentRequestCount(concurrentRequestCount);
            boolean useBufferedStream = len < BUFFERED_STREAM_THRESHHOLD;
            final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file);
            try {
                blob.upload(in, len, null, options, null);
                LOG.debug("Blob created. identifier={} length={} duration={} buffered={}", key, len, (System.currentTimeMillis() - start), useBufferedStream);
            } finally {
                in.close();
            }
            return;
        }
        blob.downloadAttributes();
        if (blob.getProperties().getLength() != len) {
            throw new DataStoreException("Length Collision. identifier=" + key + " new length=" + len + " old length=" + blob.getProperties().getLength());
        }
        LOG.trace("Blob already exists. identifier={} lastModified={}", key, blob.getProperties().getLastModified().getTime());
        blob.startCopy(blob);
        //TODO: better way of updating lastModified (use custom metadata?)
        if (!waitForCopy(blob)) {
            throw new DataStoreException(String.format("Cannot update lastModified for blob. identifier=%s status=%s", key, blob.getCopyState().getStatusDescription()));
        }
        LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key, blob.getProperties().getLastModified().getTime(), (System.currentTimeMillis() - start));
    } catch (StorageException e) {
        LOG.info("Error writing blob. identifier={}", key, e);
        throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
    } catch (URISyntaxException | IOException e) {
        LOG.debug("Error writing blob. identifier={}", key, e);
        throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
    } catch (InterruptedException e) {
        LOG.debug("Error writing blob. identifier={}", key, e);
        throw new DataStoreException(String.format("Cannot copy blob. identifier=%s", key), e);
    } finally {
        if (null != contextClassLoader) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}