Examples with WebHdfsParamsBuilder com.microsoft.azure.hdinsight.sdk.storage.webhdfs.WebHdfsParamsBuilder used on opensource projects

Example 1 with WebHdfsParamsBuilder

use of com.microsoft.azure.hdinsight.sdk.storage.webhdfs.WebHdfsParamsBuilder in project azure-tools-for-java by Microsoft.

the class WebHDFSDeploy method getArtifactUploadedPath.

@Nullable
public String getArtifactUploadedPath(String rootPath) throws URISyntaxException {
    final List<NameValuePair> params = new WebHdfsParamsBuilder("OPEN").build();
    final URIBuilder uriBuilder = new URIBuilder(rootPath);
    uriBuilder.addParameters(params);
    return uriBuilder.build().toString();
}

Example 2 with WebHdfsParamsBuilder

use of com.microsoft.azure.hdinsight.sdk.storage.webhdfs.WebHdfsParamsBuilder in project azure-tools-for-java by Microsoft.

the class JobUtils method deployArtifact.

public static Observable<String> deployArtifact(@NotNull SparkBatchSubmission submission, @NotNull String destinationRootPath, @NotNull String artifactPath) {
    return Observable.fromCallable(() -> {
        final File file = new File(artifactPath);
        final String webHdfsUploadPath = destinationRootPath + file.getName();
        final String redirectUri;
        List<NameValuePair> params = new WebHdfsParamsBuilder("CREATE").setOverwrite("true").build();
        URIBuilder uriBuilder = new URIBuilder(webHdfsUploadPath);
        uriBuilder.addParameters(params);
        HttpUriRequest req = RequestBuilder.put(uriBuilder.build()).build();
        final CloseableHttpClient httpclient = submission.getHttpClient();
        try (final CloseableHttpResponse response = httpclient.execute(req)) {
            // two steps to upload via webhdfs
            // 1.put request the get 307 redirect uri from response
            // 2.put redirect request with file content as setEntity
            redirectUri = response.getFirstHeader("Location").getValue();
            if (StringUtils.isBlank(redirectUri)) {
                throw new UnknownServiceException("can not get valid redirect uri using webhdfs");
            }
        } catch (final Exception ex) {
            throw new UnknownServiceException("using webhdfs encounter problem:" + ex.toString());
        }
        final InputStreamEntity reqEntity = new InputStreamEntity(new FileInputStream(file), -1, ContentType.APPLICATION_OCTET_STREAM);
        reqEntity.setChunked(true);
        final BufferedHttpEntity reqEntityBuf = new BufferedHttpEntity(reqEntity);
        // setup url with redirect url and entity ,config 100 continue to header
        req = RequestBuilder.put(redirectUri).setEntity(reqEntityBuf).setConfig(RequestConfig.custom().setExpectContinueEnabled(true).build()).build();
        // execute put request
        try (final CloseableHttpResponse putResp = httpclient.execute(req)) {
            params = new WebHdfsParamsBuilder("OPEN").build();
            uriBuilder = new URIBuilder(webHdfsUploadPath);
            uriBuilder.addParameters(params);
            // return get file uri
            return uriBuilder.build().toString();
        }
    });
}