Examples with HistoryFileInfo org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo used on opensource projects

Example 26 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistoryParsing method testHistoryParsingForFailedAttempts.

@Test(timeout = 30000)
public void testHistoryParsingForFailedAttempts() throws Exception {
    LOG.info("STARTING testHistoryParsingForFailedAttempts");
    try {
        Configuration conf = new Configuration();
        conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
        RackResolver.init(conf);
        MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(), true);
        app.submit(conf);
        Job job = app.getContext().getAllJobs().values().iterator().next();
        JobId jobId = job.getID();
        app.waitForState(job, JobState.SUCCEEDED);
        // make sure all events are flushed
        app.waitForState(Service.STATE.STOPPED);
        JobHistory jobHistory = new JobHistory();
        jobHistory.init(conf);
        HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
        JobHistoryParser parser;
        JobInfo jobInfo;
        synchronized (fileInfo) {
            Path historyFilePath = fileInfo.getHistoryFile();
            FSDataInputStream in = null;
            FileContext fc = null;
            try {
                fc = FileContext.getFileContext(conf);
                in = fc.open(fc.makeQualified(historyFilePath));
            } catch (IOException ioe) {
                LOG.info("Can not open history file: " + historyFilePath, ioe);
                throw (new Exception("Can not open History File"));
            }
            parser = new JobHistoryParser(in);
            jobInfo = parser.parse();
        }
        Exception parseException = parser.getParseException();
        Assert.assertNull("Caught an expected exception " + parseException, parseException);
        int noOffailedAttempts = 0;
        Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
        for (Task task : job.getTasks().values()) {
            TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
            for (TaskAttempt taskAttempt : task.getAttempts().values()) {
                TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
                // Verify rack-name for all task attempts
                Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
                if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
                    noOffailedAttempts++;
                }
            }
        }
        Assert.assertEquals("No of Failed tasks doesn't match.", 2, noOffailedAttempts);
    } finally {
        LOG.info("FINISHED testHistoryParsingForFailedAttempts");
    }
}

Example 27 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistory method testLoadJobErrorCases.

@Test
public void testLoadJobErrorCases() throws IOException {
    HistoryFileManager historyManager = mock(HistoryFileManager.class);
    jobHistory = spy(new JobHistory());
    doReturn(historyManager).when(jobHistory).createHistoryFileManager();
    Configuration conf = new Configuration();
    // Set the cache threshold to 50 tasks
    conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
    jobHistory.init(conf);
    jobHistory.start();
    CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
    assertTrue(storage.getUseLoadedTasksCache());
    assertEquals(storage.getLoadedTasksCacheSize(), 50);
    // Create jobs for bad fileInfo results
    Job[] jobs = new Job[4];
    JobId[] jobIds = new JobId[4];
    for (int i = 0; i < jobs.length; i++) {
        jobs[i] = mock(Job.class);
        jobIds[i] = mock(JobId.class);
        when(jobs[i].getID()).thenReturn(jobIds[i]);
        when(jobs[i].getTotalMaps()).thenReturn(10);
        when(jobs[i].getTotalReduces()).thenReturn(2);
    }
    HistoryFileInfo loadJobException = mock(HistoryFileInfo.class);
    when(loadJobException.loadJob()).thenThrow(new IOException("History file not found"));
    when(historyManager.getFileInfo(jobIds[0])).thenThrow(new IOException(""));
    when(historyManager.getFileInfo(jobIds[1])).thenReturn(null);
    when(historyManager.getFileInfo(jobIds[2])).thenReturn(loadJobException);
    try {
        storage.getFullJob(jobIds[0]);
        fail("Did not get expected YarnRuntimeException for getFileInfo() throwing IOException");
    } catch (YarnRuntimeException e) {
    // Expected
    }
    // fileInfo==null should return null
    Job job = storage.getFullJob(jobIds[1]);
    assertNull(job);
    try {
        storage.getFullJob(jobIds[2]);
        fail("Did not get expected YarnRuntimeException for fileInfo.loadJob() throwing IOException");
    } catch (YarnRuntimeException e) {
    // Expected
    }
}

Example 28 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistory method testRefreshLoadedJobCache.

@Test
public void testRefreshLoadedJobCache() throws Exception {
    HistoryFileManager historyManager = mock(HistoryFileManager.class);
    jobHistory = spy(new JobHistory());
    doReturn(historyManager).when(jobHistory).createHistoryFileManager();
    Configuration conf = new Configuration();
    // Set the cache size to 2
    conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 2);
    jobHistory.init(conf);
    jobHistory.start();
    CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
    assertFalse(storage.getUseLoadedTasksCache());
    Job[] jobs = new Job[3];
    JobId[] jobIds = new JobId[3];
    for (int i = 0; i < 3; i++) {
        jobs[i] = mock(Job.class);
        jobIds[i] = mock(JobId.class);
        when(jobs[i].getID()).thenReturn(jobIds[i]);
    }
    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
    when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]);
    // getFullJob will put the job in the cache if it isn't there
    for (int i = 0; i < 3; i++) {
        storage.getFullJob(jobs[i].getID());
    }
    Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
    // Verify some jobs are stored in the cache.  Hard to predict eviction
    // in Guava version.
    assertTrue(jobCache.size() > 0);
    // Setting cache size to 3
    conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE, 3);
    doReturn(conf).when(storage).createConf();
    when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]);
    jobHistory.refreshLoadedJobCache();
    for (int i = 0; i < 3; i++) {
        storage.getFullJob(jobs[i].getID());
    }
    jobCache = storage.getLoadedJobCache();
    // Verify some jobs are stored in the cache.  Hard to predict eviction
    // in Guava version.
    assertTrue(jobCache.size() > 0);
}

Example 29 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistory method testTasksCacheLimit.

@Test
public void testTasksCacheLimit() throws Exception {
    HistoryFileManager historyManager = mock(HistoryFileManager.class);
    jobHistory = spy(new JobHistory());
    doReturn(historyManager).when(jobHistory).createHistoryFileManager();
    Configuration conf = new Configuration();
    // Set the cache threshold to 50 tasks
    conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 50);
    jobHistory.init(conf);
    jobHistory.start();
    CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
    assertTrue(storage.getUseLoadedTasksCache());
    assertEquals(storage.getLoadedTasksCacheSize(), 50);
    // Create a bunch of smaller jobs (<< 50 tasks)
    Job[] jobs = new Job[10];
    JobId[] jobIds = new JobId[10];
    for (int i = 0; i < jobs.length; i++) {
        jobs[i] = mock(Job.class);
        jobIds[i] = mock(JobId.class);
        when(jobs[i].getID()).thenReturn(jobIds[i]);
        when(jobs[i].getTotalMaps()).thenReturn(10);
        when(jobs[i].getTotalReduces()).thenReturn(2);
    }
    // Create some large jobs that forces task-based cache flushing
    Job[] lgJobs = new Job[3];
    JobId[] lgJobIds = new JobId[3];
    for (int i = 0; i < lgJobs.length; i++) {
        lgJobs[i] = mock(Job.class);
        lgJobIds[i] = mock(JobId.class);
        when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
        when(lgJobs[i].getTotalMaps()).thenReturn(2000);
        when(lgJobs[i].getTotalReduces()).thenReturn(10);
    }
    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
    when(fileInfo.loadJob()).thenReturn(jobs[0]).thenReturn(jobs[1]).thenReturn(jobs[2]).thenReturn(jobs[3]).thenReturn(jobs[4]).thenReturn(jobs[5]).thenReturn(jobs[6]).thenReturn(jobs[7]).thenReturn(jobs[8]).thenReturn(jobs[9]).thenReturn(lgJobs[0]).thenReturn(lgJobs[1]).thenReturn(lgJobs[2]);
    // getFullJob will put the job in the cache if it isn't there
    Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
    for (int i = 0; i < jobs.length; i++) {
        storage.getFullJob(jobs[i].getID());
    }
    long prevSize = jobCache.size();
    // gets reduced in size.
    for (int i = 0; i < lgJobs.length; i++) {
        storage.getFullJob(lgJobs[i].getID());
    }
    assertTrue(jobCache.size() < prevSize);
}

Example 30 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistory method testJobCacheLimitLargerThanMax.

@Test
public void testJobCacheLimitLargerThanMax() throws Exception {
    HistoryFileManager historyManager = mock(HistoryFileManager.class);
    JobHistory jobHistory = spy(new JobHistory());
    doReturn(historyManager).when(jobHistory).createHistoryFileManager();
    Configuration conf = new Configuration();
    // Set the cache threshold to 50 tasks
    conf.setInt(JHAdminConfig.MR_HISTORY_LOADED_TASKS_CACHE_SIZE, 500);
    jobHistory.init(conf);
    jobHistory.start();
    CachedHistoryStorage storage = spy((CachedHistoryStorage) jobHistory.getHistoryStorage());
    assertTrue(storage.getUseLoadedTasksCache());
    assertEquals(storage.getLoadedTasksCacheSize(), 500);
    // Create a bunch of large jobs (>> 50 tasks)
    Job[] lgJobs = new Job[10];
    JobId[] lgJobIds = new JobId[10];
    for (int i = 0; i < lgJobs.length; i++) {
        lgJobs[i] = mock(Job.class);
        lgJobIds[i] = mock(JobId.class);
        when(lgJobs[i].getID()).thenReturn(lgJobIds[i]);
        when(lgJobs[i].getTotalMaps()).thenReturn(700);
        when(lgJobs[i].getTotalReduces()).thenReturn(50);
    }
    HistoryFileInfo fileInfo = mock(HistoryFileInfo.class);
    when(historyManager.getFileInfo(any(JobId.class))).thenReturn(fileInfo);
    when(fileInfo.loadJob()).thenReturn(lgJobs[0]).thenReturn(lgJobs[1]).thenReturn(lgJobs[2]).thenReturn(lgJobs[3]).thenReturn(lgJobs[4]).thenReturn(lgJobs[5]).thenReturn(lgJobs[6]).thenReturn(lgJobs[7]).thenReturn(lgJobs[8]).thenReturn(lgJobs[9]);
    // getFullJob will put the job in the cache if it isn't there
    Cache<JobId, Job> jobCache = storage.getLoadedJobCache();
    long[] cacheSize = new long[10];
    for (int i = 0; i < lgJobs.length; i++) {
        storage.getFullJob(lgJobs[i].getID());
        assertTrue(jobCache.size() > 0);
    }
}