Examples with MRRSleepJob org.apache.tez.mapreduce.examples.MRRSleepJob used on opensource projects

Example 1 with MRRSleepJob

use of org.apache.tez.mapreduce.examples.MRRSleepJob in project tez by apache.

the class TestMRRJobs method testFailingJob.

@Test(timeout = 60000)
public void testFailingJob() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testFailingJob().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;
    }
    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);
    Job job = sleepJob.createJob(1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
    job.setJarByClass(MRRSleepJob.class);
    // speed up failures
    job.setMaxMapAttempts(1);
    job.getConfiguration().setBoolean(MRRSleepJob.MAP_FATAL_ERROR, true);
    job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "*");
    job.submit();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertFalse(succeeded);
    Assert.assertEquals(JobStatus.State.FAILED, job.getJobState());
// FIXME once counters and task progress can be obtained properly
// TODO verify failed task diagnostics
}

Example 2 with MRRSleepJob

use of org.apache.tez.mapreduce.examples.MRRSleepJob in project tez by apache.

the class TestMRRJobs method testMRRSleepJobWithCompression.

@Test(timeout = 60000)
public void testMRRSleepJobWithCompression() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testMRRSleepJobWithCompression().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;
    }
    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);
    Job job = sleepJob.createJob(1, 1, 2, 1, 1, 1, 1, 1, 1, 1);
    job.setJarByClass(MRRSleepJob.class);
    // speed up failures
    job.setMaxMapAttempts(1);
    // enable compression
    job.getConfiguration().setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
    job.getConfiguration().set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, DefaultCodec.class.getName());
    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.contains(jobId.substring(jobId.indexOf("_"))));
// FIXME once counters and task progress can be obtained properly
// TODO use dag client to test counters and task progress?
// what about completed jobs?
}

Example 3 with MRRSleepJob

use of org.apache.tez.mapreduce.examples.MRRSleepJob in project tez by apache.

the class TestMRRJobs method testFailingAttempt.

@Test(timeout = 60000)
public void testFailingAttempt() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testFailingAttempt().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;
    }
    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);
    Job job = sleepJob.createJob(1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
    job.setJarByClass(MRRSleepJob.class);
    // speed up failures
    job.setMaxMapAttempts(3);
    job.getConfiguration().setBoolean(MRRSleepJob.MAP_THROW_ERROR, true);
    job.getConfiguration().set(MRRSleepJob.MAP_ERROR_TASK_IDS, "0");
    job.submit();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
// FIXME once counters and task progress can be obtained properly
// TODO verify failed task diagnostics
}

Example 4 with MRRSleepJob

use of org.apache.tez.mapreduce.examples.MRRSleepJob in project tez by apache.

the class TestMRRJobs method testMRRSleepJob.

@Test(timeout = 60000)
public void testMRRSleepJob() throws IOException, InterruptedException, ClassNotFoundException {
    LOG.info("\n\n\nStarting testMRRSleepJob().");
    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;
    }
    Configuration sleepConf = new Configuration(mrrTezCluster.getConfig());
    MRRSleepJob sleepJob = new MRRSleepJob();
    sleepJob.setConf(sleepConf);
    Job job = sleepJob.createJob(1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
    job.setJarByClass(MRRSleepJob.class);
    // speed up failures
    job.setMaxMapAttempts(1);
    job.submit();
    String trackingUrl = job.getTrackingURL();
    String jobId = job.getJobID().toString();
    boolean succeeded = job.waitForCompletion(true);
    Assert.assertTrue(succeeded);
    Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
    // There's one bug in YARN that there may be some suffix at the end of trackingURL (YARN-2246)
    // After TEZ-1961, the tracking will change from http://localhost:53419/proxy/application_1430963524753_0005
    // to http://localhost:53419/proxy/application_1430963524753_0005/ui/
    // So here use String#contains to verify.
    Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.contains(jobId.substring(jobId.indexOf("_"))));
// FIXME once counters and task progress can be obtained properly
// TODO use dag client to test counters and task progress?
// what about completed jobs?
}