Examples with HttpURLConnection java.net.HttpURLConnection used on opensource projects

Example 51 with HttpURLConnection

use of java.net.HttpURLConnection in project hadoop by apache.

the class TestShuffleHandler method testMapFileAccess.

/**
   * Validate the ownership of the map-output files being pulled in. The
   * local-file-system owner of the file should match the user component in the
   *
   * @throws Exception exception
   */
@Test(timeout = 100000)
public void testMapFileAccess() throws IOException {
    // This will run only in NativeIO is enabled as SecureIOUtils need it
    assumeTrue(NativeIO.isAvailable());
    Configuration conf = new Configuration();
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
    ApplicationId appId = ApplicationId.newInstance(12345, 1);
    LOG.info(appId.toString());
    String appAttemptId = "attempt_12345_1_m_1_0";
    String user = "randomUser";
    String reducerId = "0";
    List<File> fileMap = new ArrayList<File>();
    createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap);
    ShuffleHandler shuffleHandler = new ShuffleHandler() {

        @Override
        protected Shuffle getShuffle(Configuration conf) {
            // replace the shuffle handler with one stubbed for testing
            return new Shuffle(conf) {

                @Override
                protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException {
                // Do nothing.
                }
            };
        }
    };
    shuffleHandler.init(conf);
    try {
        shuffleHandler.start();
        DataOutputBuffer outputBuffer = new DataOutputBuffer();
        outputBuffer.reset();
        Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService"));
        jt.write(outputBuffer);
        shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));
        URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
        conn.connect();
        byte[] byteArr = new byte[10000];
        try {
            DataInputStream is = new DataInputStream(conn.getInputStream());
            is.readFully(byteArr);
        } catch (EOFException e) {
        // ignore
        }
        // Retrieve file owner name
        FileInputStream is = new FileInputStream(fileMap.get(0));
        String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner();
        is.close();
        String message = "Owner '" + owner + "' for path " + fileMap.get(0).getAbsolutePath() + " did not match expected owner '" + user + "'";
        Assert.assertTrue((new String(byteArr)).contains(message));
    } finally {
        shuffleHandler.stop();
        FileUtil.fullyDelete(absLogDir);
    }
}

Example 52 with HttpURLConnection

use of java.net.HttpURLConnection in project hadoop by apache.

the class TestAzureFileSystemErrorConditions method testTransientErrorOnCommitBlockList.

@Test
public void testTransientErrorOnCommitBlockList() throws Exception {
    // Need to do this test against a live storage account
    AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount.create();
    assumeNotNull(testAccount);
    try {
        NativeAzureFileSystem fs = testAccount.getFileSystem();
        injectTransientError(fs, new ConnectionRecognizer() {

            @Override
            public boolean isTargetConnection(HttpURLConnection connection) {
                return connection.getRequestMethod().equals("PUT") && connection.getURL().getQuery() != null && connection.getURL().getQuery().contains("blocklist");
            }
        });
        Path testFile = new Path("/a/b");
        writeAllThreeFile(fs, testFile);
        readAllThreeFile(fs, testFile);
    } finally {
        testAccount.cleanup();
    }
}

Example 53 with HttpURLConnection

use of java.net.HttpURLConnection in project hadoop by apache.

the class TestRMFailover method testWebAppProxyInStandAloneMode.

@Test
public void testWebAppProxyInStandAloneMode() throws YarnException, InterruptedException, IOException {
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
    WebAppProxyServer webAppProxyServer = new WebAppProxyServer();
    try {
        conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099");
        cluster.init(conf);
        cluster.start();
        getAdminService(0).transitionToActive(req);
        assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
        verifyConnections();
        webAppProxyServer.init(conf);
        // Start webAppProxyServer
        Assert.assertEquals(STATE.INITED, webAppProxyServer.getServiceState());
        webAppProxyServer.start();
        Assert.assertEquals(STATE.STARTED, webAppProxyServer.getServiceState());
        // send httpRequest with fakeApplicationId
        // expect to get "Not Found" response and 404 response code
        URL wrongUrl = new URL("http://0.0.0.0:9099/proxy/" + fakeAppId);
        HttpURLConnection proxyConn = (HttpURLConnection) wrongUrl.openConnection();
        proxyConn.connect();
        verifyResponse(proxyConn);
        explicitFailover();
        verifyConnections();
        proxyConn.connect();
        verifyResponse(proxyConn);
    } finally {
        webAppProxyServer.stop();
    }
}

Example 54 with HttpURLConnection

use of java.net.HttpURLConnection in project hadoop by apache.

the class TestRMFailover method testEmbeddedWebAppProxy.

@Test
public void testEmbeddedWebAppProxy() throws YarnException, InterruptedException, IOException {
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
    cluster.init(conf);
    cluster.start();
    assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
    verifyConnections();
    // send httpRequest with fakeApplicationId
    // expect to get "Not Found" response and 404 response code
    URL wrongUrl = new URL("http://0.0.0.0:18088/proxy/" + fakeAppId);
    HttpURLConnection proxyConn = (HttpURLConnection) wrongUrl.openConnection();
    proxyConn.connect();
    verifyResponse(proxyConn);
    explicitFailover();
    verifyConnections();
    proxyConn.connect();
    verifyResponse(proxyConn);
}

Example 55 with HttpURLConnection

use of java.net.HttpURLConnection in project hadoop by apache.

the class TestRMWebServicesHttpStaticUserPermissions method testWebServiceAccess.

// Test that the http static user can't submit or kill apps
// when secure mode is turned on
@Test
public void testWebServiceAccess() throws Exception {
    ApplicationSubmissionContextInfo app = new ApplicationSubmissionContextInfo();
    String appid = "application_123_0";
    app.setApplicationId(appid);
    String submitAppRequestBody = TestRMWebServicesDelegationTokenAuthentication.getMarshalledAppInfo(app);
    URL url = new URL("http://localhost:8088/ws/v1/cluster/apps");
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    // we should be access the apps page with the static user
    TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "GET", "", "");
    try {
        conn.getInputStream();
        assertEquals(Status.OK.getStatusCode(), conn.getResponseCode());
    } catch (IOException e) {
        fail("Got " + conn.getResponseCode() + " instead of 200 accessing " + url.toString());
    }
    conn.disconnect();
    // new-application, submit app and kill should fail with
    // forbidden
    Map<String, Helper> urlRequestMap = new HashMap<String, Helper>();
    String killAppRequestBody = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n" + "<appstate>\n" + "  <state>KILLED</state>\n" + "</appstate>";
    urlRequestMap.put("http://localhost:8088/ws/v1/cluster/apps", new Helper("POST", submitAppRequestBody));
    urlRequestMap.put("http://localhost:8088/ws/v1/cluster/apps/new-application", new Helper("POST", ""));
    urlRequestMap.put("http://localhost:8088/ws/v1/cluster/apps/app_123_1/state", new Helper("PUT", killAppRequestBody));
    for (Map.Entry<String, Helper> entry : urlRequestMap.entrySet()) {
        URL reqURL = new URL(entry.getKey());
        conn = (HttpURLConnection) reqURL.openConnection();
        String method = entry.getValue().method;
        String body = entry.getValue().requestBody;
        TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, method, "application/xml", body);
        try {
            conn.getInputStream();
            fail("Request " + entry.getKey() + "succeeded but should have failed");
        } catch (IOException e) {
            assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode());
            InputStream errorStream = conn.getErrorStream();
            String error = "";
            BufferedReader reader = new BufferedReader(new InputStreamReader(errorStream, "UTF8"));
            for (String line; (line = reader.readLine()) != null; ) {
                error += line;
            }
            reader.close();
            errorStream.close();
            assertEquals("The default static user cannot carry out this operation.", error);
        }
        conn.disconnect();
    }
}