Examples with HDStorageAccount com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount used on opensource projects

Example 1 with HDStorageAccount

use of com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount in project azure-tools-for-java by Microsoft.

the class SparkSubmitHelper method uploadFileToHDFS.

public String uploadFileToHDFS(/*Project project,*/
String localFile, IHDIStorageAccount storageAccount, String defaultContainerName, String uploadFolderPath) throws Exception {
    final File file = new File(localFile);
    if (storageAccount.getAccountType() == StorageAccountTypeEnum.BLOB) {
        try (FileInputStream fileInputStream = new FileInputStream(file)) {
            try (BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
                final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                    @Override
                    public Void call(Long uploadedBytes) throws Exception {
                        double progress = ((double) uploadedBytes) / file.length();
                        return null;
                    }
                };
                HDStorageAccount blobStorageAccount = (HDStorageAccount) storageAccount;
                BlobContainer defaultContainer = getSparkClusterDefaultContainer(blobStorageAccount, defaultContainerName);
                String path = String.format("SparkSubmission/%s/%s", uploadFolderPath, file.getName());
                String uploadedPath = String.format("wasb://%s@%s/%s", defaultContainerName, blobStorageAccount.getFullStorageBlobName(), path);
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Blob Storage Account %s ...", localFile, uploadedPath));
                StorageClientSDKManager.getManager().uploadBlobFileContent(blobStorageAccount.getConnectionString(), defaultContainer, path, bufferedInputStream, callable, 1024 * 1024, file.length());
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
                return uploadedPath;
            }
        }
    } else if (storageAccount.getAccountType() == StorageAccountTypeEnum.ADLS) {
        String uploadPath = String.format("adl://%s.azuredatalakestore.net/%s/%s", storageAccount.getName(), storageAccount.getDefaultContainerOrRootPath(), "SparkSubmission");
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Data Lake Store %s ...", localFile, uploadPath));
        String uploadedPath = StreamUtil.uploadArtifactToADLS(file, storageAccount, uploadFolderPath);
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
        return uploadedPath;
    } else {
        throw new UnsupportedOperationException("unkown storage account type");
    }
}

Example 2 with HDStorageAccount

use of com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount in project azure-tools-for-java by Microsoft.

the class AddHDInsightAdditionalClusterImpl method getStorageAccount.

public static HDStorageAccount getStorageAccount(String clusterName, String storageName, String storageKey, String userName, String password) throws HDIException, AzureCmdException {
    String responseMessage = getMessageByAmbari(clusterName, userName, password);
    responseMessage = responseMessage.replace(" ", "");
    if (StringHelper.isNullOrWhiteSpace(responseMessage)) {
        throw new HDIException("Failed to get storage account");
    }
    Matcher matcher = PATTERN_DEFAULT_STORAGE.matcher(responseMessage);
    String defaultContainer = "";
    if (matcher.find()) {
        defaultContainer = matcher.group(1);
    }
    if (StringHelper.isNullOrWhiteSpace(defaultContainer)) {
        throw new HDIException("Failed to get default container for storage account");
    }
    HDStorageAccount account = new HDStorageAccount(null, storageName + BLOB_URL_SUFFIX, storageKey, true, defaultContainer);
    //getting container to check the storage key is correct or not
    try {
        StorageClientSDKManager.getManager().getBlobContainers(account.getConnectionString());
    } catch (AzureCmdException e) {
        throw new AzureCmdException("Invalid Storage Key");
    }
    return account;
}

Example 3 with HDStorageAccount

use of com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount in project azure-tools-for-java by Microsoft.

the class AddHDInsightAdditionalClusterImpl method getStorageAccountsFromResponseMessage.

//not using for now.
private static List<ClientStorageAccount> getStorageAccountsFromResponseMessage(String responseMessage) throws StorageAccountResolveException {
    responseMessage = responseMessage.replace(" ", "");
    Matcher matcher = PATTERN_DEFAULT_STORAGE.matcher(responseMessage);
    String defaultStorageName = "";
    try {
        if (matcher.find()) {
            String str = matcher.group();
            defaultStorageName = str.split("[@.]")[2];
        }
    } catch (Exception e) {
        throw new StorageAccountResolveException();
    }
    matcher = PATTER_STORAGE_KEY.matcher(responseMessage);
    HashMap<String, String> storageKeysMap = new HashMap<String, String>();
    while (matcher.find()) {
        String str = matcher.group();
        String[] strs = str.replace("\"", "").split(":");
        String storageName = strs[0].split("\\.")[4];
        storageKeysMap.put(storageName, strs[1]);
    }
    if (StringHelper.isNullOrWhiteSpace(defaultStorageName) || !storageKeysMap.containsKey(defaultStorageName)) {
        throw new StorageAccountResolveException();
    }
    List<ClientStorageAccount> storageAccounts = new ArrayList<ClientStorageAccount>();
    storageAccounts.add(new HDStorageAccount(null, defaultStorageName, storageKeysMap.get(defaultStorageName), false, null));
    for (String storageName : storageKeysMap.keySet()) {
        if (!storageName.equals(defaultStorageName)) {
            storageAccounts.add(new HDStorageAccount(null, storageName, storageKeysMap.get(storageName), false, null));
        }
    }
    return storageAccounts;
}

Example 4 with HDStorageAccount

use of com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount in project azure-tools-for-java by Microsoft.

the class StorageAccountNode method refreshItems.

@Override
protected void refreshItems() throws AzureCmdException {
    //TelemetryManager.postEvent(TelemetryCommon.HDInsightExplorerStorageAccountExpand, null, null);
    if (storageAccount.getAccountType() == StorageAccountTypeEnum.BLOB) {
        HDStorageAccount blobStorageAccount = (HDStorageAccount) storageAccount;
        String defaultContainer = blobStorageAccount.getDefaultContainer();
        List<BlobContainer> containerList = StorageClientSDKManager.getManager().getBlobContainers(((HDStorageAccount) storageAccount).getConnectionString());
        for (BlobContainer blobContainer : containerList) {
            addChildNode(new BlobContainerNode(this, blobStorageAccount, blobContainer, !StringHelper.isNullOrWhiteSpace(defaultContainer) && defaultContainer.equals(blobContainer.getName())));
        }
    } else if (storageAccount.getAccountType() == StorageAccountTypeEnum.ADLS) {
    // TODO：　adls support
    }
}

Example 5 with HDStorageAccount

use of com.microsoft.azure.hdinsight.sdk.storage.HDStorageAccount in project azure-tools-for-java by Microsoft.

the class ClusterDetail method getAdditionalStorageAccounts.

private List<HDStorageAccount> getAdditionalStorageAccounts(Map<String, String> coresiteMap) {
    if (coresiteMap.size() <= 2) {
        return null;
    }
    List<HDStorageAccount> storageAccounts = new ArrayList<>();
    for (Map.Entry<String, String> entry : coresiteMap.entrySet()) {
        if (entry.getKey().toLowerCase().equals(DefaultFS) || entry.getKey().toLowerCase().equals(FSDefaultName)) {
            continue;
        }
        if (entry.getKey().contains(StorageAccountKeyPrefix)) {
            HDStorageAccount account = new HDStorageAccount(this, entry.getKey().substring(StorageAccountKeyPrefix.length()), entry.getValue(), false, null);
            storageAccounts.add(account);
        }
    }
    return storageAccounts;
}