Examples with MRClientProtocol org.apache.hadoop.mapreduce.v2.api.MRClientProtocol used on opensource projects

Example 1 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testRMDownRestoreForJobStatusBeforeGetAMReport.

@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced1"))).thenThrow(new java.lang.reflect.UndeclaredThrowableException(new IOException("Connection refuced2"))).thenReturn(getFinishedApplicationReport());
        ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
        JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
        verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
        Assert.assertNotNull(jobStatus);
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 2 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testRemoteExceptionFromHistoryServer.

@Test
public void testRemoteExceptionFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(new IOException("Job ID doesnot Exist"));
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
    try {
        clientServiceDelegate.getJobStatus(oldJobId);
        Assert.fail("Invoke should throw exception after retries.");
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist"));
    }
}

Example 3 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testJobReportFromHistoryServer.

@Test
public void testJobReportFromHistoryServer() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}

Example 4 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testReconnectOnAMRestart.

@Test
public void testReconnectOnAMRestart() throws IOException {
    //as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
        return;
    }
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
        when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn(getRunningApplicationReport("am1", 78)).thenReturn(getRunningApplicationReport(null, 0)).thenReturn(getRunningApplicationReport(null, 0)).thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
        throw new IOException(e);
    }
    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport()).thenReturn(MRBuilderUtils.newJobReport(jobId, "jobName-firstGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, ""));
    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(jobReportResponse1).thenThrow(new RuntimeException("AM is down!"));
    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport()).thenReturn(MRBuilderUtils.newJobReport(jobId, "jobName-secondGen", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, ""));
    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class))).thenReturn(jobReportResponse2);
    ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy).doReturn(secondGenAMProxy).when(clientServiceDelegate).instantiateAMProxy(any(InetSocketAddress.class));
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
}

Example 5 with MRClientProtocol

use of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol in project hadoop by apache.

the class TestClientServiceDelegate method testUnknownAppInRM.

@Test
public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
}