Examples with CloudBlob com.microsoft.azure.storage.blob.CloudBlob used on opensource projects

Example 6 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project jackrabbit-oak by apache.

the class AzureBlobStoreBackend method getAllMetadataRecords.

@Override
public List<DataRecord> getAllMetadataRecords(String prefix) {
    if (null == prefix) {
        throw new NullPointerException("prefix");
    }
    long start = System.currentTimeMillis();
    final List<DataRecord> records = Lists.newArrayList();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
        for (ListBlobItem item : metaDir.listBlobs(prefix)) {
            if (item instanceof CloudBlob) {
                CloudBlob blob = (CloudBlob) item;
                records.add(new AzureBlobStoreDataRecord(this, connectionString, containerName, new DataIdentifier(stripMetaKeyPrefix(blob.getName())), blob.getProperties().getLastModified().getTime(), blob.getProperties().getLength(), true));
            }
        }
        LOG.debug("Metadata records read. recordsRead={} metadataFolder={} duration={}", records.size(), prefix, (System.currentTimeMillis() - start));
    } catch (StorageException e) {
        LOG.info("Error reading all metadata records. metadataFolder={}", prefix, e);
    } catch (DataStoreException | URISyntaxException e) {
        LOG.debug("Error reading all metadata records. metadataFolder={}", prefix, e);
    } finally {
        if (null != contextClassLoader) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return records;
}

Example 7 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.

the class FetchAzureBlobStorage method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
    String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
    AtomicReference<Exception> storedException = new AtomicReference<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        final Map<String, String> attributes = new HashMap<>();
        final CloudBlob blob = container.getBlockBlobReference(blobPath);
        // TODO - we may be able do fancier things with ranges and
        // distribution of download over threads, investigate
        flowFile = session.write(flowFile, os -> {
            try {
                blob.download(os);
            } catch (StorageException e) {
                storedException.set(e);
                throw new IOException(e);
            }
        });
        long length = blob.getProperties().getLength();
        attributes.put("azure.length", String.valueOf(length));
        if (!attributes.isEmpty()) {
            flowFile = session.putAllAttributes(flowFile, attributes);
        }
        session.transfer(flowFile, REL_SUCCESS);
        final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
    } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) {
        if (e instanceof ProcessException && storedException.get() == null) {
            throw (ProcessException) e;
        } else {
            Exception failureException = Optional.ofNullable(storedException.get()).orElse(e);
            getLogger().error("Failure to fetch Azure blob {}", new Object[] { blobPath }, failureException);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
    }
}

Example 8 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.

the class ListAzureBlobStorage method performListing.

@Override
protected List<BlobInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions().getValue();
    String prefix = context.getProperty(PROP_PREFIX).evaluateAttributeExpressions().getValue();
    if (prefix == null) {
        prefix = "";
    }
    final List<BlobInfo> listing = new ArrayList<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, null)) {
            if (blob instanceof CloudBlob) {
                CloudBlob cloudBlob = (CloudBlob) blob;
                BlobProperties properties = cloudBlob.getProperties();
                StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri();
                Builder builder = new BlobInfo.Builder().primaryUri(uri.getPrimaryUri().toString()).blobName(cloudBlob.getName()).containerName(containerName).contentType(properties.getContentType()).contentLanguage(properties.getContentLanguage()).etag(properties.getEtag()).lastModifiedTime(properties.getLastModified().getTime()).length(properties.getLength());
                if (uri.getSecondaryUri() != null) {
                    builder.secondaryUri(uri.getSecondaryUri().toString());
                }
                if (blob instanceof CloudBlockBlob) {
                    builder.blobType(AzureStorageUtils.BLOCK);
                } else {
                    builder.blobType(AzureStorageUtils.PAGE);
                }
                listing.add(builder.build());
            }
        }
    } catch (Throwable t) {
        throw new IOException(ExceptionUtils.getRootCause(t));
    }
    return listing;
}

Example 9 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.

the class PutAzureBlobStorage method onTrigger.

public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
    String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
    AtomicReference<Exception> storedException = new AtomicReference<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        CloudBlob blob = container.getBlockBlobReference(blobPath);
        final Map<String, String> attributes = new HashMap<>();
        long length = flowFile.getSize();
        session.read(flowFile, rawIn -> {
            InputStream in = rawIn;
            if (!(in instanceof BufferedInputStream)) {
                // do not double-wrap
                in = new BufferedInputStream(rawIn);
            }
            try {
                blob.upload(in, length);
                BlobProperties properties = blob.getProperties();
                attributes.put("azure.container", containerName);
                attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString());
                attributes.put("azure.etag", properties.getEtag());
                attributes.put("azure.length", String.valueOf(length));
                attributes.put("azure.timestamp", String.valueOf(properties.getLastModified()));
            } catch (StorageException | URISyntaxException e) {
                storedException.set(e);
                throw new IOException(e);
            }
        });
        if (!attributes.isEmpty()) {
            flowFile = session.putAllAttributes(flowFile, attributes);
        }
        session.transfer(flowFile, REL_SUCCESS);
        final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
    } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) {
        if (e instanceof ProcessException && storedException.get() == null) {
            throw (ProcessException) e;
        } else {
            Exception failureException = Optional.ofNullable(storedException.get()).orElse(e);
            getLogger().error("Failed to put Azure blob {}", new Object[] { blobPath }, failureException);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
    }
}

Example 10 with CloudBlob

use of com.microsoft.azure.storage.blob.CloudBlob in project nifi by apache.

the class AbstractAzureBlobStorageIT method uploadBlob.

protected void uploadBlob(String containerName, String filePath) throws URISyntaxException, StorageException, InvalidKeyException, IOException {
    CloudBlobContainer container = AzureTestUtil.getContainer(containerName);
    CloudBlob blob = container.getBlockBlobReference(SAMPLE_BLOB_NAME);
    blob.uploadFromFile(filePath);
}