Examples with ADLSGen2FSOperation com.microsoft.azure.hdinsight.sdk.storage.adlsgen2.ADLSGen2FSOperation used on opensource projects

Example 1 with ADLSGen2FSOperation

use of com.microsoft.azure.hdinsight.sdk.storage.adlsgen2.ADLSGen2FSOperation in project azure-tools-for-java by Microsoft.

the class ADLSGen2Deploy method deploy.

@Override
public Observable<String> deploy(File src, Observer<SparkLogLine> logSubject) {
    // four steps to upload via adls gen2 rest api
    // 1.put request to create new dir
    // 2.put request to create new file(artifact) which is empty
    // 3.patch request to append data to file
    // 4.patch request to flush data to file
    final URI destURI = getUploadDir();
    // remove request / end otherwise invalid url response
    final String destStr = destURI.toString();
    final String dirPath = destStr.endsWith("/") ? destStr.substring(0, destStr.length() - 1) : destStr;
    final String filePath = String.format("%s/%s", dirPath, src.getName());
    final ADLSGen2FSOperation op = new ADLSGen2FSOperation(this.http);
    return op.createDir(dirPath, "0755").onErrorReturn(err -> {
        if (err.getMessage() != null && (err.getMessage().contains(String.valueOf(HttpStatus.SC_FORBIDDEN)) || err.getMessage().contains(String.valueOf(HttpStatus.SC_NOT_FOUND)))) {
            // Sample destinationRootPath: https://accountName.dfs.core.windows.net/fsName/SparkSubmission/
            String fileSystemRootPath = UriUtil.normalizeWithSlashEnding(URI.create(destinationRootPath)).resolve("../").toString();
            String errorMessage = String.format("Failed to create folder %s when uploading Spark application artifacts with error: %s. %s", dirPath, err.getMessage(), getForbiddenErrorHints(fileSystemRootPath));
            throw new IllegalArgumentException(errorMessage);
        } else {
            throw Exceptions.propagate(err);
        }
    }).doOnNext(ignore -> log().info(String.format("Create filesystem %s successfully.", dirPath))).flatMap(ignore -> op.createFile(filePath, "0755")).flatMap(ignore -> op.uploadData(filePath, src)).doOnNext(ignore -> log().info(String.format("Append data to file %s successfully.", filePath))).map(ignored -> AbfsUri.parse(filePath).getUri().toString());
}