Examples with MRClientProtocol org.apache.hadoop.mapreduce.v2.api.MRClientProtocol used on opensource projects

Example 26 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testCountersFromHistoryServer.

@Test
public void testCountersFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getCounters(getCountersRequest())).thenReturn(getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
    Counters counters = TypeConverter.toYarn(clientServiceDelegate.getJobCounters(oldJobId));
    Assert.assertNotNull(counters);
    Assert.assertEquals(1001, counters.getCounterGroup("dummyCounters").getCounter("dummyCounter").getValue());
}

Example 27 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testAMAccessDisabled.

@Test
public void testAMAccessDisabled() throws IOException {
    //test only applicable when AM not reachable
    if (isAMReachableFromClient) {
        return;
    }
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponseFromHistoryServer());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(getRunningApplicationReport("am1", 78)).thenReturn(getRunningApplicationReport("am1", 78)).thenReturn(getRunningApplicationReport("am1", 78)).thenReturn(getFinishedApplicationReport());
    } catch (YarnException e) {
        throw new IOException(e);
    }
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
}

Example 28 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestJHSSecurity method testDelegationToken.

@Test
public void testDelegationToken() throws IOException, InterruptedException {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    final YarnConfiguration conf = new YarnConfiguration(new JobConf());
    // Just a random principle
    conf.set(JHAdminConfig.MR_HISTORY_PRINCIPAL, "RandomOrc/localhost@apache.org");
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    final long initialInterval = 10000l;
    final long maxLifetime = 20000l;
    final long renewInterval = 10000l;
    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
        jobHistoryServer = new JobHistoryServer() {

            protected void doSecureLogin(Configuration conf) throws IOException {
            // no keytab based login
            }

            ;

            @Override
            protected JHSDelegationTokenSecretManager createJHSSecretManager(Configuration conf, HistoryServerStateStoreService store) {
                return new JHSDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval, 3600000, store);
            }

            @Override
            protected HistoryClientService createHistoryClientService() {
                return new HistoryClientService(historyContext, this.jhsDTSecretManager) {

                    @Override
                    protected void initializeWebApp(Configuration conf) {
                    // Don't need it, skip.;
                    }
                };
            }
        };
        //      final JobHistoryServer jobHistoryServer = jhServer;
        jobHistoryServer.init(conf);
        jobHistoryServer.start();
        final MRClientProtocol hsService = jobHistoryServer.getClientService().getClientHandler();
        // Fake the authentication-method
        UserGroupInformation loggedInUser = UserGroupInformation.createRemoteUser("testrenewer@APACHE.ORG");
        Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
        // Default realm is APACHE.ORG
        loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
        Token token = getDelegationToken(loggedInUser, hsService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to JHS using the delegation token
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "TheDarkLord", conf);
        GetJobReportRequest jobReportRequest = Records.newRecord(GetJobReportRequest.class);
        jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
        }
        // Renew after 50% of token age.
        while (System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
            Thread.sleep(500l);
        }
        long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
        long renewalTime = System.currentTimeMillis();
        LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
        // Wait for first expiry, but before renewed expiry.
        while (System.currentTimeMillis() > tokenFetchTime + initialInterval && System.currentTimeMillis() < nextExpTime) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        // Valid token because of renewal.
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
        }
        // Wait for expiry.
        while (System.currentTimeMillis() < renewalTime + renewInterval) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
        // Token should have expired.      
        try {
            clientUsingDT.getJobReport(jobReportRequest);
            fail("Should not have succeeded with an expired token");
        } catch (IOException e) {
            assertTrue(e.getCause().getMessage().contains("is expired"));
        }
        // Stop the existing proxy, start another.
        if (clientUsingDT != null) {
            //        RPC.stopProxy(clientUsingDT);
            clientUsingDT = null;
        }
        token = getDelegationToken(loggedInUser, hsService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to HSService using the delegation token
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "loginuser2", conf);
        try {
            clientUsingDT.getJobReport(jobReportRequest);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        }
        cancelDelegationToken(loggedInUser, hsService, token);
        // Testing the token with different renewer to cancel the token
        Token tokenWithDifferentRenewer = getDelegationToken(loggedInUser, hsService, "yarn");
        cancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
        if (clientUsingDT != null) {
            //        RPC.stopProxy(clientUsingDT);
            clientUsingDT = null;
        }
        // Creating a new connection.
        clientUsingDT = getMRClientProtocol(token, jobHistoryServer.getClientService().getBindAddress(), "loginuser2", conf);
        LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
        // Verify cancellation worked.
        try {
            clientUsingDT.getJobReport(jobReportRequest);
            fail("Should not have succeeded with a cancelled delegation token");
        } catch (IOException e) {
        }
    } finally {
        jobHistoryServer.stop();
    }
}

Example 29 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project cdap by caskdata.

the class JobHistoryServerTokenUtils method obtainToken.

/**
   * Gets a JHS delegation token and stores it in the given Credentials.
   *
   * @return the same Credentials instance as the one given in parameter.
   */
public static Credentials obtainToken(Configuration configuration, Credentials credentials) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return credentials;
    }
    String historyServerAddress = configuration.get("mapreduce.jobhistory.address");
    HostAndPort hostAndPort = HostAndPort.fromString(historyServerAddress);
    try {
        ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(configuration));
        MRClientCache clientCache = new MRClientCache(configuration, resourceMgrDelegate);
        MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
        GetDelegationTokenRequest request = new GetDelegationTokenRequestPBImpl();
        request.setRenewer(YarnUtils.getYarnTokenRenewer(configuration));
        InetSocketAddress address = new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort());
        Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(hsProxy.getDelegationToken(request).getDelegationToken(), address);
        credentials.addToken(new Text(token.getService()), token);
        LOG.debug("Adding JobHistoryServer delegation token {}.", token);
        return credentials;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}