Examples with MRClientProtocol org.apache.hadoop.mapreduce.v2.api.MRClientProtocol used on opensource projects

Example 6 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testUnknownAppInRM.

@Test
public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
}

Example 7 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testNoRetryOnAMAuthorizationException.

@Test
public void testNoRetryOnAMAuthorizationException() throws Exception {
    if (!isAMReachableFromClient) {
        return;
    }
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(getRunningApplicationReport("am1", 78));
    // throw authorization exception on first invocation
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class))).thenThrow(new AuthorizationException("Denied"));
    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rm, oldJobId, null) {

        @Override
        MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr) throws IOException {
            super.instantiateAMProxy(serviceAddr);
            return amProxy;
        }
    };
    try {
        clientServiceDelegate.getJobStatus(oldJobId);
        Assert.fail("Exception should be thrown upon AuthorizationException");
    } catch (IOException e) {
        Assert.assertEquals(AuthorizationException.class.getName() + ": Denied", e.getMessage());
    }
    // assert maxClientRetry is not decremented.
    Assert.assertEquals(conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES), clientServiceDelegate.getMaxClientRetry());
    verify(amProxy, times(1)).getJobReport(any(GetJobReportRequest.class));
}

Example 8 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testRetriesOnConnectionFailure.

@Test
public void testRetriesOnConnectionFailure() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(new RuntimeException("1")).thenThrow(new RuntimeException("2")).thenReturn(getJobReportResponse());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class));
}

Example 9 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class ClientServiceDelegate method invoke.

private synchronized Object invoke(String method, Class argClass, Object args) throws IOException {
    Method methodOb = null;
    try {
        methodOb = MRClientProtocol.class.getMethod(method, argClass);
    } catch (SecurityException e) {
        throw new YarnRuntimeException(e);
    } catch (NoSuchMethodException e) {
        throw new YarnRuntimeException("Method name mismatch", e);
    }
    maxClientRetry = this.conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
    IOException lastException = null;
    while (maxClientRetry > 0) {
        MRClientProtocol MRClientProxy = null;
        try {
            MRClientProxy = getProxy();
            return methodOb.invoke(MRClientProxy, args);
        } catch (InvocationTargetException e) {
            // Will not throw out YarnException anymore
            LOG.debug("Failed to contact AM/History for job " + jobId + " retrying..", e.getTargetException());
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            if (e.getCause() instanceof AuthorizationException) {
                throw new IOException(e.getTargetException());
            }
            // for its AM to be restarted.
            if (!usingAMProxy.get()) {
                maxClientRetry--;
            }
            usingAMProxy.set(false);
            lastException = new IOException(e.getTargetException());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        } catch (Exception e) {
            LOG.debug("Failed to contact AM/History for job " + jobId + "  Will retry..", e);
            // Force reconnection by setting the proxy to null.
            realProxy = null;
            // RM shutdown
            maxClientRetry--;
            lastException = new IOException(e.getMessage());
            try {
                Thread.sleep(100);
            } catch (InterruptedException ie) {
                LOG.warn("ClientServiceDelegate invoke call interrupted", ie);
                throw new YarnRuntimeException(ie);
            }
        }
    }
    throw lastException;
}

Example 10 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestYARNRunner method testGetHSDelegationToken.

@Test(timeout = 20000)
public void testGetHSDelegationToken() throws Exception {
    try {
        Configuration conf = new Configuration();
        // Setup mock service
        InetSocketAddress mockRmAddress = new InetSocketAddress("localhost", 4444);
        Text rmTokenSevice = SecurityUtil.buildTokenService(mockRmAddress);
        InetSocketAddress mockHsAddress = new InetSocketAddress("localhost", 9200);
        Text hsTokenSevice = SecurityUtil.buildTokenService(mockHsAddress);
        // Setup mock rm token
        RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new Text("owner"), new Text("renewer"), new Text("real"));
        Token<RMDelegationTokenIdentifier> token = new Token<RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.getKind(), rmTokenSevice);
        token.setKind(RMDelegationTokenIdentifier.KIND_NAME);
        // Setup mock history token
        org.apache.hadoop.yarn.api.records.Token historyToken = org.apache.hadoop.yarn.api.records.Token.newInstance(new byte[0], MRDelegationTokenIdentifier.KIND_NAME.toString(), new byte[0], hsTokenSevice.toString());
        GetDelegationTokenResponse getDtResponse = Records.newRecord(GetDelegationTokenResponse.class);
        getDtResponse.setDelegationToken(historyToken);
        // mock services
        MRClientProtocol mockHsProxy = mock(MRClientProtocol.class);
        doReturn(mockHsAddress).when(mockHsProxy).getConnectAddress();
        doReturn(getDtResponse).when(mockHsProxy).getDelegationToken(any(GetDelegationTokenRequest.class));
        ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
        doReturn(rmTokenSevice).when(rmDelegate).getRMDelegationTokenService();
        ClientCache clientCache = mock(ClientCache.class);
        doReturn(mockHsProxy).when(clientCache).getInitializedHSProxy();
        Credentials creds = new Credentials();
        YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache);
        // No HS token if no RM token
        yarnRunner.addHistoryToken(creds);
        verify(mockHsProxy, times(0)).getDelegationToken(any(GetDelegationTokenRequest.class));
        // No HS token if RM token, but secirity disabled.
        creds.addToken(new Text("rmdt"), token);
        yarnRunner.addHistoryToken(creds);
        verify(mockHsProxy, times(0)).getDelegationToken(any(GetDelegationTokenRequest.class));
        conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
        UserGroupInformation.setConfiguration(conf);
        creds = new Credentials();
        // No HS token if no RM token, security enabled
        yarnRunner.addHistoryToken(creds);
        verify(mockHsProxy, times(0)).getDelegationToken(any(GetDelegationTokenRequest.class));
        // HS token if RM token present, security enabled
        creds.addToken(new Text("rmdt"), token);
        yarnRunner.addHistoryToken(creds);
        verify(mockHsProxy, times(1)).getDelegationToken(any(GetDelegationTokenRequest.class));
        // No additional call to get HS token if RM and HS token present
        yarnRunner.addHistoryToken(creds);
        verify(mockHsProxy, times(1)).getDelegationToken(any(GetDelegationTokenRequest.class));
    } finally {
        // Back to defaults.
        UserGroupInformation.setConfiguration(new Configuration());
    }
}