Examples with CallableSingleArg com.microsoft.tooling.msservices.helpers.CallableSingleArg used on opensource projects

Example 1 with CallableSingleArg

use of com.microsoft.tooling.msservices.helpers.CallableSingleArg in project azure-tools-for-java by Microsoft.

the class SparkSubmitHelper method uploadFileToHDFS.

public String uploadFileToHDFS(/*Project project,*/
String localFile, IHDIStorageAccount storageAccount, String defaultContainerName, String uploadFolderPath) throws Exception {
    final File file = new File(localFile);
    if (storageAccount.getAccountType() == StorageAccountTypeEnum.BLOB) {
        try (FileInputStream fileInputStream = new FileInputStream(file)) {
            try (BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
                final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                    @Override
                    public Void call(Long uploadedBytes) throws Exception {
                        double progress = ((double) uploadedBytes) / file.length();
                        return null;
                    }
                };
                HDStorageAccount blobStorageAccount = (HDStorageAccount) storageAccount;
                BlobContainer defaultContainer = getSparkClusterDefaultContainer(blobStorageAccount, defaultContainerName);
                String path = String.format("SparkSubmission/%s/%s", uploadFolderPath, file.getName());
                String uploadedPath = String.format("wasb://%s@%s/%s", defaultContainerName, blobStorageAccount.getFullStorageBlobName(), path);
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Blob Storage Account %s ...", localFile, uploadedPath));
                StorageClientSDKManager.getManager().uploadBlobFileContent(blobStorageAccount.getConnectionString(), defaultContainer, path, bufferedInputStream, callable, 1024 * 1024, file.length());
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
                return uploadedPath;
            }
        }
    } else if (storageAccount.getAccountType() == StorageAccountTypeEnum.ADLS) {
        String uploadPath = String.format("adl://%s.azuredatalakestore.net/%s/%s", storageAccount.getName(), storageAccount.getDefaultContainerOrRootPath(), "SparkSubmission");
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Data Lake Store %s ...", localFile, uploadPath));
        String uploadedPath = StreamUtil.uploadArtifactToADLS(file, storageAccount, uploadFolderPath);
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
        return uploadedPath;
    } else {
        throw new UnsupportedOperationException("unkown storage account type");
    }
}

Example 2 with CallableSingleArg

use of com.microsoft.tooling.msservices.helpers.CallableSingleArg in project azure-tools-for-java by Microsoft.

the class SparkSubmitHelper method uploadFileToADL.

public String uploadFileToADL(/* Project project, */
String localFile, IHDIStorageAccount storageAccount, String defaultContainerName, String uploadFolderPath) throws Exception {
    final File file = new File(localFile);
    if (storageAccount.getAccountType() == StorageAccountType.BLOB) {
        try (FileInputStream fileInputStream = new FileInputStream(file)) {
            try (BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
                final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                    @Override
                    public Void call(Long uploadedBytes) throws Exception {
                        double progress = ((double) uploadedBytes) / file.length();
                        return null;
                    }
                };
                HDStorageAccount blobStorageAccount = (HDStorageAccount) storageAccount;
                BlobContainer defaultContainer = getSparkClusterDefaultContainer(blobStorageAccount, defaultContainerName);
                String path = String.format("SparkSubmission/%s/%s", uploadFolderPath, file.getName());
                String uploadedPath = String.format("wasbs://%s@%s/%s", defaultContainerName, blobStorageAccount.getFullStorageBlobName(), path);
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Blob Storage Account %s ...", localFile, uploadedPath));
                StorageClientSDKManager.getManager().uploadBlobFileContent(blobStorageAccount.getConnectionString(), defaultContainer, path, bufferedInputStream, callable, 1024 * 1024, file.length());
                HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
                return uploadedPath;
            }
        }
    } else if (storageAccount.getAccountType() == StorageAccountType.ADLS) {
        String uploadPath = String.format("adl://%s.azuredatalakestore.net/%s/%s", storageAccount.getName(), storageAccount.getDefaultContainerOrRootPath(), "SparkSubmission");
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Begin uploading file %s to Azure Data Lake Store %s ...", localFile, uploadPath));
        String uploadedPath = StreamUtil.uploadArtifactToADLS(file, storageAccount, uploadFolderPath);
        HDInsightUtil.showInfoOnSubmissionMessageWindow(String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
        return uploadedPath;
    } else {
        throw new UnsupportedOperationException("unkown storage account type");
    }
}

Example 3 with CallableSingleArg

use of com.microsoft.tooling.msservices.helpers.CallableSingleArg in project azure-tools-for-java by Microsoft.

the class BlobExplorerFileEditor method uploadFile.

private void uploadFile(final String path, final File selectedFile) {
    Job job = new Job("Uploading blob...") {

        @Override
        protected IStatus run(final IProgressMonitor monitor) {
            monitor.beginTask("Uploading blob...", IProgressMonitor.UNKNOWN);
            try {
                final BlobDirectory blobDirectory = directoryQueue.peekLast();
                final BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(selectedFile));
                monitor.subTask("0% uploaded");
                try {
                    final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                        @Override
                        public Void call(Long uploadedBytes) throws Exception {
                            double progress = ((double) uploadedBytes) / selectedFile.length();
                            monitor.worked((int) (100 * progress));
                            monitor.subTask(String.format("%s%% uploaded", (int) (progress * 100)));
                            return null;
                        }
                    };
                    try {
                        StorageClientSDKManager.getManager().uploadBlobFileContent(connectionString, blobContainer, path, bufferedInputStream, callable, 1024 * 1024, selectedFile.length());
                    } catch (AzureCmdException e) {
                        e.printStackTrace();
                    } finally {
                        try {
                            bufferedInputStream.close();
                        } catch (IOException ignored) {
                        }
                    }
                    if (monitor.isCanceled()) {
                        // future.cancel(true);
                        bufferedInputStream.close();
                        for (BlobItem blobItem : StorageClientSDKManager.getManager().getBlobItems(connectionString, blobDirectory)) {
                            if (blobItem instanceof BlobFile && blobItem.getPath().equals(path)) {
                                StorageClientSDKManager.getManager().deleteBlobFile(connectionString, (BlobFile) blobItem);
                            }
                        }
                    }
                    try {
                        directoryQueue.clear();
                        directoryQueue.addLast(StorageClientSDKManager.getManager().getRootDirectory(connectionString, blobContainer));
                        for (String pathDir : path.split("/")) {
                            for (BlobItem blobItem : StorageClientSDKManager.getManager().getBlobItems(connectionString, directoryQueue.getLast())) {
                                if (blobItem instanceof BlobDirectory && blobItem.getName().equals(pathDir)) {
                                    directoryQueue.addLast((BlobDirectory) blobItem);
                                }
                            }
                        }
                    } catch (AzureCmdException e) {
                        DefaultLoader.getUIHelper().showException("Error showing new blob", e, "Error showing new blob", false, true);
                    }
                    DefaultLoader.getIdeHelper().invokeLater(new Runnable() {

                        @Override
                        public void run() {
                            fillGrid();
                        }
                    });
                } catch (Exception e) {
                    Throwable connectionFault = e.getCause();
                    Throwable realFault = null;
                    if (connectionFault != null) {
                        realFault = connectionFault.getCause();
                    }
                    monitor.setTaskName("Error uploading Blob");
                    String message = realFault == null ? null : realFault.getMessage();
                    if (connectionFault != null && message == null) {
                        message = "Error type " + connectionFault.getClass().getName();
                    }
                    monitor.subTask((connectionFault instanceof SocketTimeoutException) ? "Connection timed out" : message);
                }
            } catch (Exception e) {
                DefaultLoader.getUIHelper().showException("Error uploading Blob", e, "Error uploading Blob", false, true);
                return Status.CANCEL_STATUS;
            } finally {
                monitor.done();
            }
            return Status.OK_STATUS;
        }
    };
    job.schedule();
}

Example 4 with CallableSingleArg

use of com.microsoft.tooling.msservices.helpers.CallableSingleArg in project azure-tools-for-java by Microsoft.

the class BlobExplorerFileEditor method uploadFile.

private void uploadFile(final String path, final File selectedFile) {
    final AzureString title = AzureOperationBundle.title("blob.upload", selectedFile, blobContainer.getName());
    AzureTaskManager.getInstance().runInBackground(new AzureTask(project, title, false, () -> {
        final ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator();
        try {
            final BlobDirectory blobDirectory = directoryQueue.peekLast();
            final BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(selectedFile));
            progressIndicator.setIndeterminate(false);
            progressIndicator.setText("Uploading blob...");
            progressIndicator.setText2("0% uploaded");
            try {
                final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                    @Override
                    public Void call(Long uploadedBytes) throws Exception {
                        double progress = ((double) uploadedBytes) / selectedFile.length();
                        progressIndicator.setFraction(progress);
                        progressIndicator.setText2(String.format("%s%% uploaded", (int) (progress * 100)));
                        return null;
                    }
                };
                Future<Void> future = ApplicationManager.getApplication().executeOnPooledThread(new Callable<Void>() {

                    @Override
                    public Void call() throws AzureCmdException {
                        try {
                            StorageClientSDKManager.getManager().uploadBlobFileContent(connectionString, blobContainer, path, bufferedInputStream, callable, 1024 * 1024, selectedFile.length());
                        } finally {
                            try {
                                bufferedInputStream.close();
                            } catch (IOException ignored) {
                            }
                        }
                        return null;
                    }
                });
                while (!future.isDone()) {
                    Thread.sleep(500);
                    progressIndicator.checkCanceled();
                    if (progressIndicator.isCanceled()) {
                        future.cancel(true);
                        bufferedInputStream.close();
                        for (BlobItem blobItem : StorageClientSDKManager.getManager().getBlobItems(connectionString, blobDirectory)) {
                            if (blobItem instanceof BlobFile && blobItem.getPath().equals(path)) {
                                StorageClientSDKManager.getManager().deleteBlobFile(connectionString, (BlobFile) blobItem);
                            }
                        }
                    }
                }
                try {
                    directoryQueue.clear();
                    directoryQueue.addLast(StorageClientSDKManager.getManager().getRootDirectory(connectionString, blobContainer));
                    for (String pathDir : path.split("/")) {
                        for (BlobItem blobItem : StorageClientSDKManager.getManager().getBlobItems(connectionString, directoryQueue.getLast())) {
                            if (blobItem instanceof BlobDirectory && blobItem.getName().equals(pathDir)) {
                                directoryQueue.addLast((BlobDirectory) blobItem);
                            }
                        }
                    }
                } catch (AzureCmdException e) {
                    String msg = "An error occurred while attempting to show new blob." + "\n" + String.format(message("webappExpMsg"), e.getMessage());
                    PluginUtil.displayErrorDialogAndLog(message("errTtl"), msg, e);
                }
                AzureTaskManager.getInstance().runLater(() -> fillGrid());
            } catch (Exception e) {
                Throwable connectionFault = e.getCause();
                Throwable realFault = null;
                if (connectionFault != null) {
                    realFault = connectionFault.getCause();
                }
                progressIndicator.setText("Error uploading Blob");
                String message = realFault == null ? null : realFault.getMessage();
                if (connectionFault != null && message == null) {
                    message = "Error type " + connectionFault.getClass().getName();
                }
                progressIndicator.setText2((connectionFault instanceof SocketTimeoutException) ? "Connection timed out" : message);
            }
        } catch (IOException e) {
            PluginUtil.displayErrorDialogAndLog(message("errTtl"), "An error occurred while attempting to upload Blob.", e);
        }
    }));
}

Example 5 with CallableSingleArg

use of com.microsoft.tooling.msservices.helpers.CallableSingleArg in project azure-tools-for-java by Microsoft.

the class SparkSubmitHelper method uploadFileToHDFS.

public String uploadFileToHDFS(Project project, String localFile, IHDIStorageAccount storageAccount, String defaultContainerName, String uploadFolderPath) throws Exception {
    final File file = new File(localFile);
    if (storageAccount.getAccountType() == StorageAccountTypeEnum.BLOB) {
        try (FileInputStream fileInputStream = new FileInputStream(file)) {
            try (BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
                final CallableSingleArg<Void, Long> callable = new CallableSingleArg<Void, Long>() {

                    @Override
                    public Void call(Long uploadedBytes) throws Exception {
                        double progress = ((double) uploadedBytes) / file.length();
                        return null;
                    }
                };
                HDStorageAccount blobStorageAccount = (HDStorageAccount) storageAccount;
                BlobContainer defaultContainer = getSparkClusterDefaultContainer(blobStorageAccount, defaultContainerName);
                String path = String.format("SparkSubmission/%s/%s", uploadFolderPath, file.getName());
                String uploadedPath = String.format("wasb://%s@%s/%s", defaultContainerName, blobStorageAccount.getFullStorageBlobName(), path);
                HDInsightUtil.showInfoOnSubmissionMessageWindow(project, String.format("Info : Begin uploading file %s to Azure Blob Storage Account %s ...", localFile, uploadedPath));
                StorageClientSDKManager.getManager().uploadBlobFileContent(blobStorageAccount.getConnectionString(), defaultContainer, path, bufferedInputStream, callable, 1024 * 1024, file.length());
                HDInsightUtil.showInfoOnSubmissionMessageWindow(project, String.format("Info : Submit file to azure blob '%s' successfully.", uploadedPath));
                return uploadedPath;
            }
        }
    } else if (storageAccount.getAccountType() == StorageAccountTypeEnum.ADLS) {
        String uploadPath = String.format("adl://%s.azuredatalakestore.net%s%s", storageAccount.getName(), storageAccount.getDefaultContainerOrRootPath(), "SparkSubmission");
        HDInsightUtil.showInfoOnSubmissionMessageWindow(project, String.format("Info : Begin uploading file %s to Azure Datalake store %s ...", localFile, uploadPath));
        String uploadedPath = StreamUtil.uploadArtifactToADLS(file, storageAccount, uploadFolderPath);
        HDInsightUtil.showInfoOnSubmissionMessageWindow(project, String.format("Info : Submit file to Azure Datalake store '%s' successfully.", uploadedPath));
        return uploadedPath;
    } else {
        throw new UnsupportedOperationException("unknown storage account type");
    }
}