Examples with MRClientService org.apache.hadoop.mapreduce.v2.app.client.MRClientService used on opensource projects

Example 1 with MRClientService

use of org.apache.hadoop.mapreduce.v2.app.client.MRClientService in project hadoop by apache.

the class TestAMWebApp method testMRWebAppRedirection.

@Test
public void testMRWebAppRedirection() throws Exception {
    String[] schemePrefix = { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
    for (String scheme : schemePrefix) {
        MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {

            @Override
            protected ClientService createClientService(AppContext context) {
                return new MRClientService(context);
            }
        };
        Configuration conf = new Configuration();
        conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
        conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme.equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name() : Policy.HTTP_ONLY.name());
        webProxyBase = "/proxy/" + app.getAppID();
        conf.set("hadoop.http.filter.initializers", TestAMFilterInitializer.class.getName());
        Job job = app.submit(conf);
        String hostPort = NetUtils.getHostPortString(((MRClientService) app.getClientService()).getWebApp().getListenerAddress());
        URL httpUrl = new URL("http://" + hostPort + "/mapreduce");
        HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
        conn.setInstanceFollowRedirects(false);
        conn.connect();
        // Because we're not calling from the proxy's address, we'll be redirected
        String expectedURL = scheme + conf.get(YarnConfiguration.PROXY_ADDRESS) + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce", true);
        Assert.assertEquals(expectedURL, conn.getHeaderField(HttpHeaders.LOCATION));
        Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY, conn.getResponseCode());
        app.waitForState(job, JobState.SUCCEEDED);
        app.verifyCompleted();
    }
}

Example 2 with MRClientService

use of org.apache.hadoop.mapreduce.v2.app.client.MRClientService in project hadoop by apache.

the class MRClientService method serviceStart.

protected void serviceStart() throws Exception {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress address = new InetSocketAddress(0);
    server = rpc.getServer(MRClientProtocol.class, protocolHandler, address, conf, appContext.getClientToAMTokenSecretManager(), conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
    // Enable service authorization?
    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
        refreshServiceAcls(conf, new MRAMPolicyProvider());
    }
    server.start();
    this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(), server.getListenerAddress().getPort());
    LOG.info("Instantiated MRClientService at " + this.bindAddress);
    try {
        // Explicitly disabling SSL for map reduce task as we can't allow MR users
        // to gain access to keystore file for opening SSL listener. We can trust
        // RM/NM to issue SSL certificates but definitely not MR-AM as it is
        // running in user-land.
        webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws").withHttpPolicy(conf, Policy.HTTP_ONLY).withPortRange(conf, MRJobConfig.MR_AM_WEBAPP_PORT_RANGE).start(new AMWebApp());
    } catch (Exception e) {
        LOG.error("Webapps failed to start. Ignoring for now:", e);
    }
    super.serviceStart();
}

Example 3 with MRClientService

use of org.apache.hadoop.mapreduce.v2.app.client.MRClientService in project hadoop by apache.

the class TestAMWebApp method testMRWebAppSSLDisabled.

@Test
public void testMRWebAppSSLDisabled() throws Exception {
    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {

        @Override
        protected ClientService createClientService(AppContext context) {
            return new MRClientService(context);
        }
    };
    Configuration conf = new Configuration();
    // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
    Job job = app.submit(conf);
    String hostPort = NetUtils.getHostPortString(((MRClientService) app.getClientService()).getWebApp().getListenerAddress());
    // http:// should be accessible
    URL httpUrl = new URL("http://" + hostPort);
    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
    InputStream in = conn.getInputStream();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    IOUtils.copyBytes(in, out, 1024);
    Assert.assertTrue(out.toString().contains("MapReduce Application"));
    // https:// is not accessible.
    URL httpsUrl = new URL("https://" + hostPort);
    try {
        HttpURLConnection httpsConn = (HttpURLConnection) httpsUrl.openConnection();
        httpsConn.getInputStream();
        Assert.fail("https:// is not accessible, expected to fail");
    } catch (Exception e) {
        Assert.assertTrue(e instanceof SSLException);
    }
    app.waitForState(job, JobState.SUCCEEDED);
    app.verifyCompleted();
}