Examples with MRClientProtocol org.apache.hadoop.mapreduce.v2.api.MRClientProtocol used on opensource projects

Example 21 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class YARNRunner method getDelegationTokenFromHS.

@VisibleForTesting
Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy) throws IOException, InterruptedException {
    GetDelegationTokenRequest request = recordFactory.newRecordInstance(GetDelegationTokenRequest.class);
    request.setRenewer(Master.getMasterPrincipal(conf));
    org.apache.hadoop.yarn.api.records.Token mrDelegationToken;
    mrDelegationToken = hsProxy.getDelegationToken(request).getDelegationToken();
    return ConverterUtils.convertFromYarn(mrDelegationToken, hsProxy.getConnectAddress());
}

Example 22 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class ClientCache method instantiateHistoryProxy.

protected MRClientProtocol instantiateHistoryProxy() throws IOException {
    final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
    if (StringUtils.isEmpty(serviceAddr)) {
        return null;
    }
    LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
    final YarnRPC rpc = YarnRPC.create(conf);
    LOG.debug("Connected to HistoryServer at: " + serviceAddr);
    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {

        @Override
        public MRClientProtocol run() {
            return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf);
        }
    });
}

Example 23 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class ClientServiceDelegate method instantiateAMProxy.

MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr) throws IOException {
    LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy = (MRClientProtocol) rpc.getProxy(MRClientProtocol.class, serviceAddr, conf);
    usingAMProxy.set(true);
    LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
    return proxy;
}

Example 24 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testRMDownForJobStatusBeforeGetAMReport.

private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries) throws IOException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced1"))).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced2"))).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced3")));
        ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
        try {
            clientServiceDelegate.getJobStatus(oldJobId);
            Assert.fail("It should throw exception after retries");
        } catch (IOException e) {
            System.out.println("fail to get job status,and e=" + e.toString());
        }
        verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 25 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testRetriesOnAMConnectionFailures.

@Test
public void testRetriesOnAMConnectionFailures() throws Exception {
    if (!isAMReachableFromClient) {
        return;
    }
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(getRunningApplicationReport("am1", 78));
    // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
    // succeed in the 5th call.
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class))).thenThrow(new RuntimeException("11")).thenThrow(new RuntimeException("22")).thenThrow(new RuntimeException("33")).thenThrow(new RuntimeException("44")).thenReturn(getJobReportResponse());
    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm, oldJobId, null) {

        @Override
        MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr) throws IOException {
            super.instantiateAMProxy(serviceAddr);
            return amProxy;
        }
    };
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    // assert maxClientRetry is not decremented.
    Assert.assertEquals(conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES), clientServiceDelegate.getMaxClientRetry());
    verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class));
}