Examples with CloudBlobClient com.microsoft.azure.storage.blob.CloudBlobClient used on opensource projects

Example 16 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project nifi by apache.

the class FetchAzureBlobStorage method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
    String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
    AtomicReference<Exception> storedException = new AtomicReference<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        final Map<String, String> attributes = new HashMap<>();
        final CloudBlob blob = container.getBlockBlobReference(blobPath);
        // TODO - we may be able do fancier things with ranges and
        // distribution of download over threads, investigate
        flowFile = session.write(flowFile, os -> {
            try {
                blob.download(os);
            } catch (StorageException e) {
                storedException.set(e);
                throw new IOException(e);
            }
        });
        long length = blob.getProperties().getLength();
        attributes.put("azure.length", String.valueOf(length));
        if (!attributes.isEmpty()) {
            flowFile = session.putAllAttributes(flowFile, attributes);
        }
        session.transfer(flowFile, REL_SUCCESS);
        final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
    } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) {
        if (e instanceof ProcessException && storedException.get() == null) {
            throw (ProcessException) e;
        } else {
            Exception failureException = Optional.ofNullable(storedException.get()).orElse(e);
            getLogger().error("Failure to fetch Azure blob {}", new Object[] { blobPath }, failureException);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
    }
}

Example 17 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project nifi by apache.

the class ListAzureBlobStorage method performListing.

@Override
protected List<BlobInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions().getValue();
    String prefix = context.getProperty(PROP_PREFIX).evaluateAttributeExpressions().getValue();
    if (prefix == null) {
        prefix = "";
    }
    final List<BlobInfo> listing = new ArrayList<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, null)) {
            if (blob instanceof CloudBlob) {
                CloudBlob cloudBlob = (CloudBlob) blob;
                BlobProperties properties = cloudBlob.getProperties();
                StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri();
                Builder builder = new BlobInfo.Builder().primaryUri(uri.getPrimaryUri().toString()).blobName(cloudBlob.getName()).containerName(containerName).contentType(properties.getContentType()).contentLanguage(properties.getContentLanguage()).etag(properties.getEtag()).lastModifiedTime(properties.getLastModified().getTime()).length(properties.getLength());
                if (uri.getSecondaryUri() != null) {
                    builder.secondaryUri(uri.getSecondaryUri().toString());
                }
                if (blob instanceof CloudBlockBlob) {
                    builder.blobType(AzureStorageUtils.BLOCK);
                } else {
                    builder.blobType(AzureStorageUtils.PAGE);
                }
                listing.add(builder.build());
            }
        }
    } catch (Throwable t) {
        throw new IOException(ExceptionUtils.getRootCause(t));
    }
    return listing;
}

Example 18 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project nifi by apache.

the class PutAzureBlobStorage method onTrigger.

public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
    String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
    AtomicReference<Exception> storedException = new AtomicReference<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        CloudBlob blob = container.getBlockBlobReference(blobPath);
        final Map<String, String> attributes = new HashMap<>();
        long length = flowFile.getSize();
        session.read(flowFile, rawIn -> {
            InputStream in = rawIn;
            if (!(in instanceof BufferedInputStream)) {
                // do not double-wrap
                in = new BufferedInputStream(rawIn);
            }
            try {
                blob.upload(in, length);
                BlobProperties properties = blob.getProperties();
                attributes.put("azure.container", containerName);
                attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString());
                attributes.put("azure.etag", properties.getEtag());
                attributes.put("azure.length", String.valueOf(length));
                attributes.put("azure.timestamp", String.valueOf(properties.getLastModified()));
            } catch (StorageException | URISyntaxException e) {
                storedException.set(e);
                throw new IOException(e);
            }
        });
        if (!attributes.isEmpty()) {
            flowFile = session.putAllAttributes(flowFile, attributes);
        }
        session.transfer(flowFile, REL_SUCCESS);
        final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
    } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) {
        if (e instanceof ProcessException && storedException.get() == null) {
            throw (ProcessException) e;
        } else {
            Exception failureException = Optional.ofNullable(storedException.get()).orElse(e);
            getLogger().error("Failed to put Azure blob {}", new Object[] { blobPath }, failureException);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
    }
}

Example 19 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project nifi by apache.

the class AzureStorageUtils method createCloudBlobClient.

/**
 * Create CloudBlobClient instance.
 * @param flowFile An incoming FlowFile can be used for NiFi Expression Language evaluation to derive
 *                 Account Name, Account Key or SAS Token. This can be null if not available.
 */
public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger, FlowFile flowFile) {
    final String accountName;
    final String accountKey;
    final String sasToken;
    if (flowFile == null) {
        accountName = context.getProperty(AzureStorageUtils.ACCOUNT_NAME).evaluateAttributeExpressions().getValue();
        accountKey = context.getProperty(AzureStorageUtils.ACCOUNT_KEY).evaluateAttributeExpressions().getValue();
        sasToken = context.getProperty(AzureStorageUtils.PROP_SAS_TOKEN).evaluateAttributeExpressions().getValue();
    } else {
        accountName = context.getProperty(AzureStorageUtils.ACCOUNT_NAME).evaluateAttributeExpressions(flowFile).getValue();
        accountKey = context.getProperty(AzureStorageUtils.ACCOUNT_KEY).evaluateAttributeExpressions(flowFile).getValue();
        sasToken = context.getProperty(AzureStorageUtils.PROP_SAS_TOKEN).evaluateAttributeExpressions(flowFile).getValue();
    }
    CloudBlobClient cloudBlobClient;
    try {
        // sas token and acct name/key have different ways of creating a secure connection (e.g. new StorageCredentialsAccountAndKey didn't work)
        if (StringUtils.isNotBlank(sasToken)) {
            String storageConnectionString = String.format(AzureStorageUtils.FORMAT_BASE_URI, accountName);
            StorageCredentials creds = new StorageCredentialsSharedAccessSignature(sasToken);
            cloudBlobClient = new CloudBlobClient(new URI(storageConnectionString), creds);
        } else {
            String blobConnString = String.format(AzureStorageUtils.FORMAT_BLOB_CONNECTION_STRING, accountName, accountKey);
            CloudStorageAccount storageAccount = CloudStorageAccount.parse(blobConnString);
            cloudBlobClient = storageAccount.createCloudBlobClient();
        }
    } catch (IllegalArgumentException | URISyntaxException e) {
        logger.error("Invalid connection string URI for '{}'", new Object[] { context.getName() }, e);
        throw new IllegalArgumentException(e);
    } catch (InvalidKeyException e) {
        logger.error("Invalid connection credentials for '{}'", new Object[] { context.getName() }, e);
        throw new IllegalArgumentException(e);
    }
    return cloudBlobClient;
}

Example 20 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project nifi by apache.

the class AzureTestUtil method getContainer.

public static CloudBlobContainer getContainer(String containerName) throws InvalidKeyException, URISyntaxException, StorageException {
    String storageConnectionString = String.format(AzureStorageUtils.FORMAT_BLOB_CONNECTION_STRING, getAccountName(), getAccountKey());
    CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
    return blobClient.getContainerReference(containerName);
}