Examples with JobStory org.apache.hadoop.tools.rumen.JobStory used on opensource projects

Example 1 with JobStory

use of org.apache.hadoop.tools.rumen.JobStory in project hadoop by apache.

the class TestSleepJob method testRandomLocation.

private void testRandomLocation(int locations, int njobs, UserGroupInformation ugi) throws Exception {
    Configuration configuration = new Configuration();
    DebugJobProducer jobProducer = new DebugJobProducer(njobs, configuration);
    Configuration jconf = GridmixTestUtils.mrvl.getConfig();
    jconf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations);
    JobStory story;
    int seq = 1;
    while ((story = jobProducer.getNextJob()) != null) {
        GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0, story, new Path("ignored"), ugi, seq++);
        gridmixJob.buildSplits(null);
        List<InputSplit> splits = new SleepJob.SleepInputFormat().getSplits(gridmixJob.getJob());
        for (InputSplit split : splits) {
            assertEquals(locations, split.getLocations().length);
        }
    }
    jobProducer.close();
}

Example 2 with JobStory

use of org.apache.hadoop.tools.rumen.JobStory in project hadoop by apache.

the class TestSleepJob method testMapTasksOnlySleepJobs.

@Test
public void testMapTasksOnlySleepJobs() throws Exception {
    Configuration configuration = GridmixTestUtils.mrvl.getConfig();
    DebugJobProducer jobProducer = new DebugJobProducer(5, configuration);
    configuration.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    JobStory story;
    int seq = 1;
    while ((story = jobProducer.getNextJob()) != null) {
        GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(configuration, 0, story, new Path("ignored"), ugi, seq++);
        gridmixJob.buildSplits(null);
        Job job = gridmixJob.call();
        assertEquals(0, job.getNumReduceTasks());
    }
    jobProducer.close();
    assertEquals(6, seq);
}

Example 3 with JobStory

use of org.apache.hadoop.tools.rumen.JobStory in project hadoop by apache.

the class DistributedCacheEmulator method buildDistCacheFilesList.

/**
   * Create the list of unique distributed cache files needed for all the
   * simulated jobs and write the list to a special file.
   * @param jsp job story producer for the trace
   * @return exit code
   * @throws IOException
   */
private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
    // Read all the jobs from the trace file and build the list of unique
    // distributed cache files.
    JobStory jobStory;
    while ((jobStory = jsp.getNextJob()) != null) {
        if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS && jobStory.getSubmissionTime() >= 0) {
            updateHDFSDistCacheFilesList(jobStory);
        }
    }
    jsp.close();
    return writeDistCacheFilesList();
}

Example 4 with JobStory

use of org.apache.hadoop.tools.rumen.JobStory in project hadoop by apache.

the class TestGridMixClasses method testCompareGridmixJob.

/*
   * test CompareGridmixJob only equals and compare
   */
@Test(timeout = 30000)
public void testCompareGridmixJob() throws Exception {
    Configuration conf = new Configuration();
    Path outRoot = new Path("target");
    JobStory jobDesc = mock(JobStory.class);
    when(jobDesc.getName()).thenReturn("JobName");
    when(jobDesc.getJobConf()).thenReturn(new JobConf(conf));
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    GridmixJob j1 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
    GridmixJob j2 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
    GridmixJob j3 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
    GridmixJob j4 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
    assertTrue(j1.equals(j2));
    assertEquals(0, j1.compareTo(j2));
    // Only one parameter matters
    assertFalse(j1.equals(j3));
    // compare id and submissionMillis
    assertEquals(-1, j1.compareTo(j3));
    assertEquals(-1, j1.compareTo(j4));
}

Example 5 with JobStory

use of org.apache.hadoop.tools.rumen.JobStory in project hadoop by apache.

the class JobFactory method getNextJobFiltered.

protected JobStory getNextJobFiltered() throws IOException {
    JobStory job = getNextJobFromTrace();
    // These jobs are not yet supported in Gridmix
    while (job != null && (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS || job.getSubmissionTime() < 0 || job.getNumberMaps() == 0)) {
        if (LOG.isDebugEnabled()) {
            List<String> reason = new ArrayList<String>();
            if (job.getOutcome() != Pre21JobHistoryConstants.Values.SUCCESS) {
                reason.add("STATE (" + job.getOutcome().name() + ")");
            }
            if (job.getSubmissionTime() < 0) {
                reason.add("SUBMISSION-TIME (" + job.getSubmissionTime() + ")");
            }
            if (job.getNumberMaps() == 0) {
                reason.add("ZERO-MAPS-JOB");
            }
            // TODO This should never happen. Probably we missed something!
            if (reason.size() == 0) {
                reason.add("N/A");
            }
            LOG.debug("Ignoring job " + job.getJobID() + " from the input trace." + " Reason: " + StringUtils.join(reason, ","));
        }
        job = getNextJobFromTrace();
    }
    return null == job ? null : new FilterJobStory(job) {

        @Override
        public TaskInfo getTaskInfo(TaskType taskType, int taskNumber) {
            TaskInfo info = this.job.getTaskInfo(taskType, taskNumber);
            if (info != null) {
                info = new MinTaskInfo(info);
            } else {
                info = new MinTaskInfo(new TaskInfo(0, 0, 0, 0, 0));
            }
            return info;
        }
    };
}