Examples with CloudBlobWrapper org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper used on opensource projects

Example 1 with CloudBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method retrieve.

@Override
public DataInputStream retrieve(String key, long startByteOffset) throws AzureException, IOException {
    try {
        // Azure storage server.
        if (null == storageInteractionLayer) {
            final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
            throw new AssertionError(errMsg);
        }
        checkContainer(ContainerAccessType.PureRead);
        // Get blob reference and open the input buffer stream.
        CloudBlobWrapper blob = getBlobReference(key);
        // Open input stream and seek to the start offset.
        InputStream in = blob.openInputStream(getDownloadOptions(), getInstrumentedContext(isConcurrentOOBAppendAllowed()));
        // Create a data input stream.
        DataInputStream inDataStream = new DataInputStream(in);
        // Skip bytes and ignore return value. This is okay
        // because if you try to skip too far you will be positioned
        // at the end and reads will not return data.
        inDataStream.skip(startByteOffset);
        return inDataStream;
    } catch (Exception e) {
        // Re-throw as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 2 with CloudBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method rename.

@Override
public void rename(String srcKey, String dstKey, boolean acquireLease, SelfRenewingLease existingLease) throws IOException {
    LOG.debug("Moving {} to {}", srcKey, dstKey);
    if (acquireLease && existingLease != null) {
        throw new IOException("Cannot acquire new lease if one already exists.");
    }
    CloudBlobWrapper srcBlob = null;
    CloudBlobWrapper dstBlob = null;
    SelfRenewingLease lease = null;
    try {
        // storage server.
        if (null == storageInteractionLayer) {
            final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri);
            throw new AssertionError(errMsg);
        }
        checkContainer(ContainerAccessType.ReadThenWrite);
        // Get the source blob and assert its existence. If the source key
        // needs to be normalized then normalize it.
        //
        srcBlob = getBlobReference(srcKey);
        if (!srcBlob.exists(getInstrumentedContext())) {
            throw new AzureException("Source blob " + srcKey + " does not exist.");
        }
        /**
       * Conditionally get a lease on the source blob to prevent other writers
       * from changing it. This is used for correctness in HBase when log files
       * are renamed. It generally should do no harm other than take a little
       * more time for other rename scenarios. When the HBase master renames a
       * log file folder, the lease locks out other writers.  This
       * prevents a region server that the master thinks is dead, but is still
       * alive, from committing additional updates.  This is different than
       * when HBase runs on HDFS, where the region server recovers the lease
       * on a log file, to gain exclusive access to it, before it splits it.
       */
        if (acquireLease) {
            lease = srcBlob.acquireLease();
        } else if (existingLease != null) {
            lease = existingLease;
        }
        // Get the destination blob. The destination key always needs to be
        // normalized.
        //
        dstBlob = getBlobReference(dstKey);
        // throttled.
        try {
            dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
        } catch (StorageException se) {
            if (se.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
                int copyBlobMinBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MIN_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL);
                int copyBlobMaxBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL);
                int copyBlobDeltaBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_BACKOFF_INTERVAL);
                int copyBlobMaxRetries = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_IO_RETRIES, DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS);
                BlobRequestOptions options = new BlobRequestOptions();
                options.setRetryPolicyFactory(new RetryExponentialRetry(copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries));
                dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
            } else {
                throw se;
            }
        }
        waitForCopyToComplete(dstBlob, getInstrumentedContext());
        safeDelete(srcBlob, lease);
    } catch (StorageException e) {
        if (e.getHttpStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
            LOG.warn("Rename: CopyBlob: StorageException: ServerBusy: Retry complete, will attempt client side copy for page blob");
            InputStream ipStream = null;
            OutputStream opStream = null;
            try {
                if (srcBlob.getProperties().getBlobType() == BlobType.PAGE_BLOB) {
                    ipStream = openInputStream(srcBlob);
                    opStream = openOutputStream(dstBlob);
                    byte[] buffer = new byte[PageBlobFormatHelpers.PAGE_SIZE];
                    int len;
                    while ((len = ipStream.read(buffer)) != -1) {
                        opStream.write(buffer, 0, len);
                    }
                    opStream.flush();
                    opStream.close();
                    ipStream.close();
                } else {
                    throw new AzureException(e);
                }
                safeDelete(srcBlob, lease);
            } catch (StorageException se) {
                LOG.warn("Rename: CopyBlob: StorageException: Failed");
                throw new AzureException(se);
            } finally {
                IOUtils.closeStream(ipStream);
                IOUtils.closeStream(opStream);
            }
        } else {
            throw new AzureException(e);
        }
    } catch (URISyntaxException e) {
        // Re-throw exception as an Azure storage exception.
        throw new AzureException(e);
    }
}

Example 3 with CloudBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method changePermissionStatus.

/**
   * Changes the permission status on the given key.
   */
@Override
public void changePermissionStatus(String key, PermissionStatus newPermission) throws AzureException {
    try {
        checkContainer(ContainerAccessType.ReadThenWrite);
        CloudBlobWrapper blob = getBlobReference(key);
        blob.downloadAttributes(getInstrumentedContext());
        storePermissionStatus(blob, newPermission);
        blob.uploadMetadata(getInstrumentedContext());
    } catch (Exception e) {
        throw new AzureException(e);
    }
}

Example 4 with CloudBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method list.

private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException {
    try {
        checkContainer(ContainerAccessType.PureRead);
        if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        // Enable flat listing option only if depth is unbounded and config
        // KEY_ENABLE_FLAT_LISTING is enabled.
        boolean enableFlatListing = false;
        if (maxListingDepth < 0 && sessionConfiguration.getBoolean(KEY_ENABLE_FLAT_LISTING, DEFAULT_ENABLE_FLAT_LISTING)) {
            enableFlatListing = true;
        }
        Iterable<ListBlobItem> objects;
        if (prefix.equals("/")) {
            objects = listRootBlobs(true, enableFlatListing);
        } else {
            objects = listRootBlobs(prefix, true, enableFlatListing);
        }
        ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>();
        for (ListBlobItem blobItem : objects) {
            //
            if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) {
                break;
            }
            if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) {
                String blobKey = null;
                CloudBlobWrapper blob = (CloudBlobWrapper) blobItem;
                BlobProperties properties = blob.getProperties();
                // Determine format of the blob name depending on whether an absolute
                // path is being used or not.
                blobKey = normalizeKey(blob);
                FileMetadata metadata;
                if (retrieveFolderAttribute(blob)) {
                    metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit);
                } else {
                    metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob));
                }
                // Add the metadata to the list, but remove any existing duplicate
                // entries first that we may have added by finding nested files.
                FileMetadata existing = getFileMetadataInList(fileMetadata, blobKey);
                if (existing != null) {
                    fileMetadata.remove(existing);
                }
                fileMetadata.add(metadata);
            } else if (blobItem instanceof CloudBlobDirectoryWrapper) {
                CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem;
                // Determine format of directory name depending on whether an absolute
                // path is being used or not.
                //
                String dirKey = normalizeKey(directory);
                // Strip the last /
                if (dirKey.endsWith(PATH_DELIMITER)) {
                    dirKey = dirKey.substring(0, dirKey.length() - 1);
                }
                // Reached the targeted listing depth. Return metadata for the
                // directory using default permissions.
                //
                // Note: Something smarter should be done about permissions. Maybe
                // inherit the permissions of the first non-directory blob.
                // Also, getting a proper value for last-modified is tricky.
                FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit);
                // there.
                if (getFileMetadataInList(fileMetadata, dirKey) == null) {
                    fileMetadata.add(directoryMetadata);
                }
                if (!enableFlatListing) {
                    // Currently at a depth of one, decrement the listing depth for
                    // sub-directories.
                    buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1);
                }
            }
        }
        // Note: Original code indicated that this may be a hack.
        priorLastKey = null;
        PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix });
        return listing;
    } catch (Exception e) {
        //
        throw new AzureException(e);
    }
}

Example 5 with CloudBlobWrapper

use of org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper in project hadoop by apache.

the class AzureNativeFileSystemStore method getBlobReference.

/**
   * This private method uses the root directory or the original container to
   * get the block blob reference depending on whether the original file system
   * object was constructed with a short- or long-form URI. If the root
   * directory is non-null the URI in the file constructor was in the long form.
   * 
   * @param aKey
   *          : a key used to query Azure for the block blob.
   * @returns blob : a reference to the Azure block blob corresponding to the
   *          key.
   * @throws URISyntaxException
   * 
   */
private CloudBlobWrapper getBlobReference(String aKey) throws StorageException, URISyntaxException {
    CloudBlobWrapper blob = null;
    if (isPageBlobKey(aKey)) {
        blob = this.container.getPageBlobReference(aKey);
    } else {
        blob = this.container.getBlockBlobReference(aKey);
        blob.setStreamMinimumReadSizeInBytes(downloadBlockSizeBytes);
        blob.setWriteBlockSizeInBytes(uploadBlockSizeBytes);
    }
    return blob;
}