Examples with CloudBlobClient com.microsoft.azure.storage.blob.CloudBlobClient used on opensource projects

Example 21 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project components by Talend.

the class AzureStorageSourceOrSink method getSchemaNames.

@Override
public List<NamedThing> getSchemaNames(RuntimeContainer container) throws IOException {
    List<NamedThing> result = new ArrayList<>();
    try {
        CloudStorageAccount storageAccount = getAzureConnection(container).getCloudStorageAccount();
        CloudBlobClient client = storageAccount.createCloudBlobClient();
        for (CloudBlobContainer c : client.listContainers()) {
            result.add(new SimpleNamedThing(c.getName(), c.getName()));
        }
    } catch (InvalidKeyException | URISyntaxException e) {
        throw new ComponentException(e);
    }
    return result;
}

Example 22 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project components by Talend.

the class AzureStorageBlobService method createContainerIfNotExist.

/**
 * This method create an azure container if it doesn't exist and set it access policy
 *
 * @param containerName : the name of the container to be created
 * @return true if the container was created, false otherwise
 */
public boolean createContainerIfNotExist(final String containerName) throws StorageException, URISyntaxException, InvalidKeyException {
    CloudBlobClient cloudBlobClient = connection.getCloudStorageAccount().createCloudBlobClient();
    CloudBlobContainer cloudBlobContainer = cloudBlobClient.getContainerReference(containerName);
    boolean containerCreated;
    try {
        containerCreated = cloudBlobContainer.createIfNotExists();
    } catch (StorageException e) {
        if (!e.getErrorCode().equals(StorageErrorCodeStrings.CONTAINER_BEING_DELETED)) {
            throw e;
        }
        LOGGER.warn(messages.getMessage("error.CONTAINER_BEING_DELETED", containerName));
        // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-container
        try {
            Thread.sleep(40000);
        } catch (InterruptedException eint) {
            LOGGER.error(messages.getMessage("error.InterruptedException"));
            throw new ComponentException(eint);
        }
        containerCreated = cloudBlobContainer.createIfNotExists();
        LOGGER.debug(messages.getMessage("debug.ContainerCreated", containerName));
    }
    return containerCreated;
}

Example 23 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project components by Talend.

the class AzureStorageBlobService method upload.

public void upload(final String containerName, final String blobName, final InputStream sourceStream, final long length) throws StorageException, IOException, URISyntaxException, InvalidKeyException {
    CloudBlobClient cloudBlobClient = connection.getCloudStorageAccount().createCloudBlobClient();
    CloudBlobContainer cloudBlobContainer = cloudBlobClient.getContainerReference(containerName);
    CloudBlockBlob blob = cloudBlobContainer.getBlockBlobReference(blobName);
    blob.upload(sourceStream, length);
}

Example 24 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project photon-model by vmware.

the class AzureStorageEnumerationAdapterService method getStorageContainersAsync.

/*
     * Get all Azure containers by storage account
     */
public void getStorageContainersAsync(StorageEnumContext context, StorageEnumStages next) {
    if (context.storageAccountIds.size() == 0) {
        logFine(() -> "No storage description available - clean up all resources");
        context.subStage = StorageEnumStages.DISASSOCIATE_RESOURCE_GROUP_STATES;
        handleSubStage(context);
        return;
    }
    Consumer<Throwable> failure = e -> {
        logWarning("Failure getting Azure storage containers [EndpointLink:%s] [Exception:%s]", context.request.endpointLink, e.getMessage());
        handleError(context, e);
    };
    PhotonModelUtils.runInExecutor(this.executorService, () -> {
        for (String id : context.storageAccountIds) {
            String storageConnectionString = context.storageConnectionStrings.get(id);
            if (storageConnectionString == null) {
                continue;
            }
            try {
                CloudStorageAccount storageAccount;
                if (AzureUtils.isAzureClientMock()) {
                    StorageCredentials credentials = StorageCredentials.tryParseCredentials(storageConnectionString);
                    storageAccount = new CloudStorageAccount(credentials, new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()), new URI(AzureUtils.getAzureBaseUri()));
                } else {
                    storageAccount = CloudStorageAccount.parse(storageConnectionString);
                }
                CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
                ResultContinuation nextContainerResults = null;
                do {
                    try {
                        ResultSegment<CloudBlobContainer> contSegment = blobClient.listContainersSegmented(null, ContainerListingDetails.NONE, getQueryResultLimit(), nextContainerResults, null, null);
                        context.apiListStorageContainers++;
                        nextContainerResults = contSegment.getContinuationToken();
                        for (CloudBlobContainer container : contSegment.getResults()) {
                            String uri = canonizeId(container.getUri().toString());
                            context.containerIds.add(uri);
                            context.storageContainers.put(uri, container);
                            ResultContinuation nextBlobResults = null;
                            do {
                                ResultSegment<ListBlobItem> blobsSegment = container.listBlobsSegmented(null, false, EnumSet.noneOf(BlobListingDetails.class), getQueryResultLimit(), nextBlobResults, null, null);
                                context.apiListBlobs++;
                                nextBlobResults = blobsSegment.getContinuationToken();
                                for (ListBlobItem blobItem : blobsSegment.getResults()) {
                                    String blobId = canonizeId(blobItem.getUri().toString());
                                    context.storageBlobs.put(blobId, blobItem);
                                    // populate mapping of blob uri and storage account for all storage
                                    // accounts as new disks can be added to already existing blobs
                                    StorageAccount blobStorageAccount = context.storageAccountMap.get(id);
                                    if (blobStorageAccount != null) {
                                        context.storageAccountBlobUriMap.put(blobId, blobStorageAccount);
                                    }
                                    context.blobIds.add(blobId);
                                }
                            } while (nextBlobResults != null);
                        }
                    } catch (StorageException storageExc) {
                        if (storageExc.getCause() instanceof UnknownHostException || StorageErrorCode.RESOURCE_NOT_FOUND.toString().equals(storageExc.getErrorCode()) || AzureConstants.RESOURCE_NOT_FOUND.equals(storageExc.getErrorCode())) {
                            String msg = "Probably trying to process a storage account/container that was " + "just deleted. Skipping it and continue with the next " + "storage account. Storage account id: [" + id + "], " + "storage account connection string: [" + storageConnectionString + "]. Error: %s";
                            logInfo(msg, Utils.toString(storageExc));
                        } else {
                            logSevere("StorageException[errorCode=%s, httpCode=%s, msg=%s, cause=%s]", storageExc.getErrorCode(), storageExc.getHttpStatusCode(), storageExc.getMessage(), storageExc.getCause() != null ? Utils.toString(storageExc.getCause()) : "n/a");
                            throw storageExc;
                        }
                    }
                } while (nextContainerResults != null);
                logFine(() -> String.format("Processing %d storage containers", context.containerIds.size()));
            } catch (Exception e) {
                handleError(context, e);
                return;
            }
        }
        context.subStage = next;
        handleSubStage(context);
    }, failure);
}

Example 25 with CloudBlobClient

use of com.microsoft.azure.storage.blob.CloudBlobClient in project cloudbreak by hortonworks.

the class AzureSetup method validateWasbFileSystem.

private void validateWasbFileSystem(FileSystem fileSystem, String fileSystemType) throws URISyntaxException, InvalidKeyException, StorageException {
    String accountName = fileSystem.getParameter(FileSystemConfiguration.ACCOUNT_NAME, String.class);
    String accountKey = fileSystem.getParameter(FileSystemConfiguration.ACCOUNT_KEY, String.class);
    String connectionString = "DefaultEndpointsProtocol=https;AccountName=" + accountName + ";AccountKey=" + accountKey;
    CloudStorageAccount storageAccount = CloudStorageAccount.parse(connectionString);
    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
    CloudBlobContainer containerReference = blobClient.getContainerReference(TEST_CONTAINER + System.nanoTime());
    try {
        containerReference.createIfNotExists();
        containerReference.delete();
    } catch (StorageException e) {
        if (e.getCause() instanceof UnknownHostException) {
            throw new CloudConnectorException("The provided account does not belong to a valid storage account");
        }
    }
}