Examples with MRAppMaster org.apache.hadoop.mapreduce.v2.app.MRAppMaster used on opensource projects

Example 11 with MRAppMaster

use of org.apache.hadoop.mapreduce.v2.app.MRAppMaster in project hadoop by apache.

the class MRAppMasterTestLaunchTime method testMRAppMasterSuccessLock.

@Test
public void testMRAppMasterSuccessLock() throws IOException, InterruptedException {
    String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
    String containerIdStr = "container_1317529182569_0004_000002_1";
    String userName = "TestAppMasterUser";
    JobConf conf = new JobConf();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(applicationAttemptIdStr);
    JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
    Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
    Path end = MRApps.getEndJobCommitSuccessFile(conf, userName, jobId);
    FileSystem fs = FileSystem.get(conf);
    fs.create(start).close();
    fs.create(end).close();
    ContainerId containerId = ContainerId.fromString(containerIdStr);
    MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis(), false, false);
    boolean caught = false;
    try {
        MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
    } catch (IOException e) {
        //The IO Exception is expected
        LOG.info("Caught expected Exception", e);
        caught = true;
    }
    assertTrue(caught);
    assertTrue(appMaster.errorHappenedShutDown);
    assertEquals(JobStateInternal.SUCCEEDED, appMaster.forcedState);
    appMaster.stop();
    // verify the final status is SUCCEEDED
    verifyFailedStatus((MRAppMasterTest) appMaster, "SUCCEEDED");
}

Example 12 with MRAppMaster

use of org.apache.hadoop.mapreduce.v2.app.MRAppMaster in project hadoop by apache.

the class MRAppMasterTestLaunchTime method testMRAppMasterFailLock.

@Test
public void testMRAppMasterFailLock() throws IOException, InterruptedException {
    String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
    String containerIdStr = "container_1317529182569_0004_000002_1";
    String userName = "TestAppMasterUser";
    JobConf conf = new JobConf();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(applicationAttemptIdStr);
    JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
    Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
    Path end = MRApps.getEndJobCommitFailureFile(conf, userName, jobId);
    FileSystem fs = FileSystem.get(conf);
    fs.create(start).close();
    fs.create(end).close();
    ContainerId containerId = ContainerId.fromString(containerIdStr);
    MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis(), false, false);
    boolean caught = false;
    try {
        MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
    } catch (IOException e) {
        //The IO Exception is expected
        LOG.info("Caught expected Exception", e);
        caught = true;
    }
    assertTrue(caught);
    assertTrue(appMaster.errorHappenedShutDown);
    assertEquals(JobStateInternal.FAILED, appMaster.forcedState);
    appMaster.stop();
    // verify the final status is FAILED
    verifyFailedStatus((MRAppMasterTest) appMaster, "FAILED");
}

Example 13 with MRAppMaster

use of org.apache.hadoop.mapreduce.v2.app.MRAppMaster in project hadoop by apache.

the class MRAppMaster method shutDownJob.

@VisibleForTesting
public void shutDownJob() {
    try {
        //if isLastAMRetry comes as true, should never set it to false
        if (!isLastAMRetry) {
            if (((JobImpl) job).getInternalState() != JobStateInternal.REBOOT) {
                LOG.info("Job finished cleanly, recording last MRAppMaster retry");
                isLastAMRetry = true;
            }
        }
        notifyIsLastAMRetry(isLastAMRetry);
        // Stop all services
        // This will also send the final report to the ResourceManager
        LOG.info("Calling stop for all the services");
        MRAppMaster.this.stop();
        if (isLastAMRetry) {
            // users and it is the last AM retry
            if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) {
                try {
                    LOG.info("Job end notification started for jobID : " + job.getReport().getJobId());
                    JobEndNotifier notifier = new JobEndNotifier();
                    notifier.setConf(getConfig());
                    JobReport report = job.getReport();
                    // from RM, so we should let users know FAILED via notifier as well
                    if (!context.hasSuccessfullyUnregistered()) {
                        report.setJobState(JobState.FAILED);
                    }
                    notifier.notify(report);
                } catch (InterruptedException ie) {
                    LOG.warn("Job end notification interrupted for jobID : " + job.getReport().getJobId(), ie);
                }
            }
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        clientService.stop();
    } catch (Throwable t) {
        LOG.warn("Graceful stop failed. Exiting.. ", t);
        exitMRAppMaster(1, t);
    }
    exitMRAppMaster(0, null);
}

Example 14 with MRAppMaster

use of org.apache.hadoop.mapreduce.v2.app.MRAppMaster in project hadoop by apache.

the class MRAppMasterTestLaunchTime method testMRAppMasterMidLock.

@Test
public void testMRAppMasterMidLock() throws IOException, InterruptedException {
    String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
    String containerIdStr = "container_1317529182569_0004_000002_1";
    String userName = "TestAppMasterUser";
    JobConf conf = new JobConf();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    conf.setInt(org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, 1);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(applicationAttemptIdStr);
    JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
    Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
    FileSystem fs = FileSystem.get(conf);
    //Create the file, but no end file so we should unregister with an error.
    fs.create(start).close();
    ContainerId containerId = ContainerId.fromString(containerIdStr);
    MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis(), false, false);
    boolean caught = false;
    try {
        MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
    } catch (IOException e) {
        //The IO Exception is expected
        LOG.info("Caught expected Exception", e);
        caught = true;
    }
    assertTrue(caught);
    assertTrue(appMaster.errorHappenedShutDown);
    assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
    appMaster.stop();
    // verify the final status is FAILED
    verifyFailedStatus((MRAppMasterTest) appMaster, "FAILED");
}

Example 15 with MRAppMaster

use of org.apache.hadoop.mapreduce.v2.app.MRAppMaster in project hadoop by apache.

the class TestStagingCleanup method testDeletionofStagingOnKillLastTry.

// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
//   @Test (timeout = 30000)
public void testDeletionofStagingOnKillLastTry() throws IOException {
    conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
    fs = mock(FileSystem.class);
    when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
    //Staging Dir exists
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    Path stagingDir = MRApps.getStagingAreaDir(conf, user);
    when(fs.exists(stagingDir)).thenReturn(true);
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    JobId jobid = recordFactory.newRecordInstance(JobId.class);
    jobid.setAppId(appId);
    ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
    //no retry
    MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
    appMaster.init(conf);
    assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
    //simulate the process being killed
    MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook(appMaster);
    hook.run();
    assertTrue("MRAppMaster isn't stopped", appMaster.isInState(Service.STATE.STOPPED));
    verify(fs).delete(stagingJobPath, true);
}