use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.
the class TestJobHistoryParsing method testHistoryParsingForFailedAttempts.
@Test(timeout = 30000)
public void testHistoryParsingForFailedAttempts() throws Exception {
LOG.info("STARTING testHistoryParsingForFailedAttempts");
try {
Configuration conf = new Configuration();
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
RackResolver.init(conf);
MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(), true);
app.submit(conf);
Job job = app.getContext().getAllJobs().values().iterator().next();
JobId jobId = job.getID();
app.waitForState(job, JobState.SUCCEEDED);
// make sure all events are flushed
app.waitForState(Service.STATE.STOPPED);
JobHistory jobHistory = new JobHistory();
jobHistory.init(conf);
HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
JobHistoryParser parser;
JobInfo jobInfo;
synchronized (fileInfo) {
Path historyFilePath = fileInfo.getHistoryFile();
FSDataInputStream in = null;
FileContext fc = null;
try {
fc = FileContext.getFileContext(conf);
in = fc.open(fc.makeQualified(historyFilePath));
} catch (IOException ioe) {
LOG.info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(in);
jobInfo = parser.parse();
}
Exception parseException = parser.getParseException();
Assert.assertNull("Caught an expected exception " + parseException, parseException);
int noOffailedAttempts = 0;
Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
for (Task task : job.getTasks().values()) {
TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
for (TaskAttempt taskAttempt : task.getAttempts().values()) {
TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
// Verify rack-name for all task attempts
Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
noOffailedAttempts++;
}
}
}
Assert.assertEquals("No of Failed tasks doesn't match.", 2, noOffailedAttempts);
} finally {
LOG.info("FINISHED testHistoryParsingForFailedAttempts");
}
}
use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.
the class TestJobHistory method testLoadJobErrorCases.
@Test
public void testLoadJobErrorCases() throws IOException {
HistoryFileManager historyManager = mock(HistoryFileManager.class);
jobHistory = spy(new JobHistory());
doReturn(historyManager).when(jobHistory).createHistoryFileManager();
Configuration conf = new Configuration();
// Set the cache threshold to 50 tasks
conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
jobHistory.init(conf);
jobHistory.start();
CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
assertTrue(storage.getUseLoadedTasksCache());
assertEquals(storage.getLoadedTasksCacheSize(), 50);
// Create jobs for bad fileInfo results
Job[] jobs = new Job[4];
JobId[] jobIds = new JobId[4];
for (int i = 0; i < jobs.length; i++) {
jobs[i] = mock(Job.class);
jobIds[i] = mock(JobId.class);
when(jobs[i].getID()).thenReturn(jobIds[i]);
when(jobs[i].getTotalMaps()).thenReturn(10);
when(jobs[i].getTotalReduces()).thenReturn(2);
}
HistoryFileInfo loadJobException = mock(HistoryFileInfo.class);
when(loadJobException.loadJob()).thenThrow(new IOException("History file not found"));
when(historyManager.getFileInfo(jobIds[0])).thenThrow(new IOException(""));
when(historyManager.getFileInfo(jobIds[1])).thenReturn(null);
when(historyManager.getFileInfo(jobIds[2])).thenReturn(loadJobException);
try {
storage.getFullJob(jobIds[0]);
fail("Did not get expected YarnRuntimeException for getFileInfo() throwing IOException");
} catch (YarnRuntimeException e) {
// Expected
}
// fileInfo==null should return null
Job job = storage.getFullJob(jobIds[1]);
assertNull(job);
try {
storage.getFullJob(jobIds[2]);
fail("Did not get expected YarnRuntimeException for fileInfo.loadJob() throwing IOException");
} catch (YarnRuntimeException e) {
// Expected
}
}
use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.
the class TestJobHistory method testRefreshLoadedJobCache.
@Test
public void testRefreshLoadedJobCache() throws Exception {
HistoryFileManager historyManager = mock(HistoryFileManager.class);
jobHistory = spy(new JobHistory());
doReturn(historyManager).when(jobHistory).createHistoryFileManager();
Configuration conf = new Configuration();
// Set the cache size to 2
conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 2);
jobHistory.init(conf);
jobHistory.start();
CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
assertFalse(storage.getUseLoadedTasksCache());
Job[] jobs = new Job[3];
JobId[] jobIds = new JobId[3];
for (int i = 0; i < 3; i++) {
jobs[i] = mock(Job.class);
jobIds[i] = mock(JobId.class);
when(jobs[i].getID()).thenReturn(jobIds[i]);
}
HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]);
// getFullJob will put the job in the cache if it isn't there
for (int i = 0; i < 3; i++) {
storage.getFullJob(jobs[i].getID());
}
Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
// Verify some jobs are stored in the cache. Hard to predict eviction
// in Guava version.
assertTrue(jobCache.size() > 0);
// Setting cache size to 3
conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 3);
doReturn(conf).when(storage).createConf();
when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]);
jobHistory.refreshLoadedJobCache();
for (int i = 0; i < 3; i++) {
storage.getFullJob(jobs[i].getID());
}
jobCache = storage.getLoadedJobCache();
// Verify some jobs are stored in the cache. Hard to predict eviction
// in Guava version.
assertTrue(jobCache.size() > 0);
}
use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.
the class TestJobHistory method testTasksCacheLimit.
@Test
public void testTasksCacheLimit() throws Exception {
HistoryFileManager historyManager = mock(HistoryFileManager.class);
jobHistory = spy(new JobHistory());
doReturn(historyManager).when(jobHistory).createHistoryFileManager();
Configuration conf = new Configuration();
// Set the cache threshold to 50 tasks
conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
jobHistory.init(conf);
jobHistory.start();
CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
assertTrue(storage.getUseLoadedTasksCache());
assertEquals(storage.getLoadedTasksCacheSize(), 50);
// Create a bunch of smaller jobs (<< 50 tasks)
Job[] jobs = new Job[10];
JobId[] jobIds = new JobId[10];
for (int i = 0; i < jobs.length; i++) {
jobs[i] = mock(Job.class);
jobIds[i] = mock(JobId.class);
when(jobs[i].getID()).thenReturn(jobIds[i]);
when(jobs[i].getTotalMaps()).thenReturn(10);
when(jobs[i].getTotalReduces()).thenReturn(2);
}
// Create some large jobs that forces task-based cache flushing
Job[] lgJobs = new Job[3];
JobId[] lgJobIds = new JobId[3];
for (int i = 0; i < lgJobs.length; i++) {
lgJobs[i] = mock(Job.class);
lgJobIds[i] = mock(JobId.class);
when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
when(lgJobs[i].getTotalMaps()).thenReturn(2000);
when(lgJobs[i].getTotalReduces()).thenReturn(10);
}
HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]).thenReturn(jobs[3]).thenReturn(jobs[4]).thenReturn(jobs[5]).thenReturn(jobs[6]).thenReturn(jobs[7]).thenReturn(jobs[8]).thenReturn(jobs[9]).thenReturn(lgJobs[0]).thenReturn(lgJobs[1]).thenReturn(lgJobs[2]);
// getFullJob will put the job in the cache if it isn't there
Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
for (int i = 0; i < jobs.length; i++) {
storage.getFullJob(jobs[i].getID());
}
long prevSize = jobCache.size();
// gets reduced in size.
for (int i = 0; i < lgJobs.length; i++) {
storage.getFullJob(lgJobs[i].getID());
}
assertTrue(jobCache.size() < prevSize);
}
use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.
the class TestJobHistory method testJobCacheLimitLargerThanMax.
@Test
public void testJobCacheLimitLargerThanMax() throws Exception {
HistoryFileManager historyManager = mock(HistoryFileManager.class);
JobHistory jobHistory = spy(new JobHistory());
doReturn(historyManager).when(jobHistory).createHistoryFileManager();
Configuration conf = new Configuration();
// Set the cache threshold to 50 tasks
conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 500);
jobHistory.init(conf);
jobHistory.start();
CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
assertTrue(storage.getUseLoadedTasksCache());
assertEquals(storage.getLoadedTasksCacheSize(), 500);
// Create a bunch of large jobs (>> 50 tasks)
Job[] lgJobs = new Job[10];
JobId[] lgJobIds = new JobId[10];
for (int i = 0; i < lgJobs.length; i++) {
lgJobs[i] = mock(Job.class);
lgJobIds[i] = mock(JobId.class);
when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
when(lgJobs[i].getTotalMaps()).thenReturn(700);
when(lgJobs[i].getTotalReduces()).thenReturn(50);
}
HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
when(fileInfo.loadJob()).thenReturn(lgJobs[0]).thenReturn(lgJobs[1]).thenReturn(lgJobs[2]).thenReturn(lgJobs[3]).thenReturn(lgJobs[4]).thenReturn(lgJobs[5]).thenReturn(lgJobs[6]).thenReturn(lgJobs[7]).thenReturn(lgJobs[8]).thenReturn(lgJobs[9]);
// getFullJob will put the job in the cache if it isn't there
Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
long[] cacheSize = new long[10];
for (int i = 0; i < lgJobs.length; i++) {
storage.getFullJob(lgJobs[i].getID());
assertTrue(jobCache.size() > 0);
}
}
Aggregations