Examples with CommitterEventHandler org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler used on opensource projects

Example 6 with CommitterEventHandler

use of org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler in project hadoop by apache.

the class TestCommitterEventHandler method testBasic.

@Test
public void testBasic() throws Exception {
    AppContext mockContext = mock(AppContext.class);
    OutputCommitter mockCommitter = mock(OutputCommitter.class);
    Clock mockClock = mock(Clock.class);
    CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(attemptid.getApplicationId()));
    WaitForItHandler waitForItHandler = new WaitForItHandler();
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
    when(mockContext.getApplicationAttemptId()).thenReturn(attemptid);
    when(mockContext.getEventHandler()).thenReturn(waitForItHandler);
    when(mockContext.getClock()).thenReturn(mockClock);
    handler.init(conf);
    handler.start();
    try {
        handler.handle(new CommitterJobCommitEvent(jobId, mockJobContext));
        String user = UserGroupInformation.getCurrentUser().getShortUserName();
        Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
        Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
        Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
        Event e = waitForItHandler.getAndClearEvent();
        assertNotNull(e);
        assertTrue(e instanceof JobCommitCompletedEvent);
        FileSystem fs = FileSystem.get(conf);
        assertTrue(startCommitFile.toString(), fs.exists(startCommitFile));
        assertTrue(endCommitSuccessFile.toString(), fs.exists(endCommitSuccessFile));
        assertFalse(endCommitFailureFile.toString(), fs.exists(endCommitFailureFile));
        verify(mockCommitter).commitJob(any(JobContext.class));
    } finally {
        handler.stop();
    }
}

Example 7 with CommitterEventHandler

use of org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler in project hadoop by apache.

the class TestJobImpl method testKilledDuringKillAbort.

@Test(timeout = 20000)
public void testKilledDuringKillAbort() throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    // not initializing dispatcher to avoid potential race condition between
    // the dispatcher thread & test thread - see MAPREDUCE-6831
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    OutputCommitter committer = new StubbedOutputCommitter() {

        @Override
        public synchronized void abortJob(JobContext jobContext, State state) throws IOException {
            while (!Thread.interrupted()) {
                try {
                    wait();
                } catch (InterruptedException e) {
                }
            }
        }
    };
    CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer);
    commitHandler.init(conf);
    commitHandler.start();
    JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
    JobId jobId = job.getID();
    job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
    assertJobState(job, JobStateInternal.INITED);
    job.handle(new JobStartEvent(jobId));
    assertJobState(job, JobStateInternal.SETUP);
    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    assertJobState(job, JobStateInternal.KILL_ABORT);
    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    assertJobState(job, JobStateInternal.KILLED);
    dispatcher.stop();
    commitHandler.stop();
}

Example 8 with CommitterEventHandler

use of org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler in project hadoop by apache.

the class TestJobImpl method testCommitJobFailsJob.

@Test(timeout = 20000)
public void testCommitJobFailsJob() throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    CyclicBarrier syncBarrier = new CyclicBarrier(2);
    OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false);
    CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer);
    commitHandler.init(conf);
    commitHandler.start();
    JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
    completeJobTasks(job);
    assertJobState(job, JobStateInternal.COMMITTING);
    // let the committer fail and verify the job fails
    syncBarrier.await();
    assertJobState(job, JobStateInternal.FAILED);
    dispatcher.stop();
    commitHandler.stop();
}

Example 9 with CommitterEventHandler

use of org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler in project hadoop by apache.

the class MRAppMaster method serviceInit.

@Override
protected void serviceInit(final Configuration conf) throws Exception {
    // create the job classloader if enabled
    createJobClassLoader(conf);
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
    initJobCredentialsAndUGI(conf);
    dispatcher = createDispatcher();
    addIfService(dispatcher);
    taskAttemptFinishingMonitor = createTaskAttemptFinishingMonitor(dispatcher.getEventHandler());
    addIfService(taskAttemptFinishingMonitor);
    context = new RunningAppContext(conf, taskAttemptFinishingMonitor);
    // Job name is the same as the app name util we support DAG of jobs
    // for an app later
    appName = conf.get(MRJobConfig.JOB_NAME, "<missing app name>");
    conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
    newApiCommitter = false;
    jobId = MRBuilderUtils.newJobId(appAttemptID.getApplicationId(), appAttemptID.getApplicationId().getId());
    int numReduceTasks = conf.getInt(MRJobConfig.NUM_REDUCES, 0);
    if ((numReduceTasks > 0 && conf.getBoolean("mapred.reducer.new-api", false)) || (numReduceTasks == 0 && conf.getBoolean("mapred.mapper.new-api", false))) {
        newApiCommitter = true;
        LOG.info("Using mapred newApiCommitter.");
    }
    boolean copyHistory = false;
    committer = createOutputCommitter(conf);
    try {
        String user = UserGroupInformation.getCurrentUser().getShortUserName();
        Path stagingDir = MRApps.getStagingAreaDir(conf, user);
        FileSystem fs = getFileSystem(conf);
        boolean stagingExists = fs.exists(stagingDir);
        Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
        boolean commitStarted = fs.exists(startCommitFile);
        Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
        boolean commitSuccess = fs.exists(endCommitSuccessFile);
        Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
        boolean commitFailure = fs.exists(endCommitFailureFile);
        if (!stagingExists) {
            isLastAMRetry = true;
            LOG.info("Attempt num: " + appAttemptID.getAttemptId() + " is last retry: " + isLastAMRetry + " because the staging dir doesn't exist.");
            errorHappenedShutDown = true;
            forcedState = JobStateInternal.ERROR;
            shutDownMessage = "Staging dir does not exist " + stagingDir;
            LOG.fatal(shutDownMessage);
        } else if (commitStarted) {
            //A commit was started so this is the last time, we just need to know
            // what result we will use to notify, and how we will unregister
            errorHappenedShutDown = true;
            isLastAMRetry = true;
            LOG.info("Attempt num: " + appAttemptID.getAttemptId() + " is last retry: " + isLastAMRetry + " because a commit was started.");
            copyHistory = true;
            if (commitSuccess) {
                shutDownMessage = "Job commit succeeded in a prior MRAppMaster attempt " + "before it crashed. Recovering.";
                forcedState = JobStateInternal.SUCCEEDED;
            } else if (commitFailure) {
                shutDownMessage = "Job commit failed in a prior MRAppMaster attempt " + "before it crashed. Not retrying.";
                forcedState = JobStateInternal.FAILED;
            } else {
                if (isCommitJobRepeatable()) {
                    // cleanup previous half done commits if committer supports
                    // repeatable job commit.
                    errorHappenedShutDown = false;
                    cleanupInterruptedCommit(conf, fs, startCommitFile);
                } else {
                    //The commit is still pending, commit error
                    shutDownMessage = "Job commit from a prior MRAppMaster attempt is " + "potentially in progress. Preventing multiple commit executions";
                    forcedState = JobStateInternal.ERROR;
                }
            }
        }
    } catch (IOException e) {
        throw new YarnRuntimeException("Error while initializing", e);
    }
    if (errorHappenedShutDown) {
        NoopEventHandler eater = new NoopEventHandler();
        //We do not have a JobEventDispatcher in this path
        dispatcher.register(JobEventType.class, eater);
        EventHandler<JobHistoryEvent> historyService = null;
        if (copyHistory) {
            historyService = createJobHistoryHandler(context);
            dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, historyService);
        } else {
            dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, eater);
        }
        if (copyHistory) {
            // Now that there's a FINISHING state for application on RM to give AMs
            // plenty of time to clean up after unregister it's safe to clean staging
            // directory after unregistering with RM. So, we start the staging-dir
            // cleaner BEFORE the ContainerAllocator so that on shut-down,
            // ContainerAllocator unregisters first and then the staging-dir cleaner
            // deletes staging directory.
            addService(createStagingDirCleaningService());
        }
        // service to allocate containers from RM (if non-uber) or to fake it (uber)
        containerAllocator = createContainerAllocator(null, context);
        addIfService(containerAllocator);
        dispatcher.register(ContainerAllocator.EventType.class, containerAllocator);
        if (copyHistory) {
            // Add the JobHistoryEventHandler last so that it is properly stopped first.
            // This will guarantee that all history-events are flushed before AM goes
            // ahead with shutdown.
            // Note: Even though JobHistoryEventHandler is started last, if any
            // component creates a JobHistoryEvent in the meanwhile, it will be just be
            // queued inside the JobHistoryEventHandler 
            addIfService(historyService);
            JobHistoryCopyService cpHist = new JobHistoryCopyService(appAttemptID, dispatcher.getEventHandler());
            addIfService(cpHist);
        }
    } else {
        //service to handle requests from JobClient
        clientService = createClientService(context);
        // Init ClientService separately so that we stop it separately, since this
        // service needs to wait some time before it stops so clients can know the
        // final states
        clientService.init(conf);
        containerAllocator = createContainerAllocator(clientService, context);
        //service to handle the output committer
        committerEventHandler = createCommitterEventHandler(context, committer);
        addIfService(committerEventHandler);
        //policy handling preemption requests from RM
        callWithJobClassLoader(conf, new Action<Void>() {

            public Void call(Configuration conf) {
                preemptionPolicy = createPreemptionPolicy(conf);
                preemptionPolicy.init(context);
                return null;
            }
        });
        //service to handle requests to TaskUmbilicalProtocol
        taskAttemptListener = createTaskAttemptListener(context, preemptionPolicy);
        addIfService(taskAttemptListener);
        //service to log job history events
        EventHandler<JobHistoryEvent> historyService = createJobHistoryHandler(context);
        dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, historyService);
        this.jobEventDispatcher = new JobEventDispatcher();
        //register the event dispatchers
        dispatcher.register(JobEventType.class, jobEventDispatcher);
        dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
        dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
        dispatcher.register(CommitterEventType.class, committerEventHandler);
        if (conf.getBoolean(MRJobConfig.MAP_SPECULATIVE, false) || conf.getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false)) {
            //optional service to speculate on task attempts' progress
            speculator = createSpeculator(conf, context);
            addIfService(speculator);
        }
        speculatorEventDispatcher = new SpeculatorEventDispatcher(conf);
        dispatcher.register(Speculator.EventType.class, speculatorEventDispatcher);
        // Now that there's a FINISHING state for application on RM to give AMs
        // plenty of time to clean up after unregister it's safe to clean staging
        // directory after unregistering with RM. So, we start the staging-dir
        // cleaner BEFORE the ContainerAllocator so that on shut-down,
        // ContainerAllocator unregisters first and then the staging-dir cleaner
        // deletes staging directory.
        addService(createStagingDirCleaningService());
        // service to allocate containers from RM (if non-uber) or to fake it (uber)
        addIfService(containerAllocator);
        dispatcher.register(ContainerAllocator.EventType.class, containerAllocator);
        // corresponding service to launch allocated containers via NodeManager
        containerLauncher = createContainerLauncher(context);
        addIfService(containerLauncher);
        dispatcher.register(ContainerLauncher.EventType.class, containerLauncher);
        // Add the JobHistoryEventHandler last so that it is properly stopped first.
        // This will guarantee that all history-events are flushed before AM goes
        // ahead with shutdown.
        // Note: Even though JobHistoryEventHandler is started last, if any
        // component creates a JobHistoryEvent in the meanwhile, it will be just be
        // queued inside the JobHistoryEventHandler 
        addIfService(historyService);
    }
    super.serviceInit(conf);
}

Example 10 with CommitterEventHandler

use of org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler in project hadoop by apache.

the class TestCommitterEventHandler method testCommitWindow.

@Test
public void testCommitWindow() throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    TestingJobEventHandler jeh = new TestingJobEventHandler();
    dispatcher.register(JobEventType.class, jeh);
    SystemClock clock = SystemClock.getInstance();
    AppContext appContext = mock(AppContext.class);
    ApplicationAttemptId attemptid = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
    when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    OutputCommitter committer = mock(OutputCommitter.class);
    TestingRMHeartbeatHandler rmhh = new TestingRMHeartbeatHandler();
    CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh);
    ceh.init(conf);
    ceh.start();
    // verify trying to commit when RM heartbeats are stale does not commit
    ceh.handle(new CommitterJobCommitEvent(null, null));
    long timeToWaitMs = 5000;
    while (rmhh.getNumCallbacks() != 1 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not register a heartbeat callback", 1, rmhh.getNumCallbacks());
    verify(committer, never()).commitJob(any(JobContext.class));
    Assert.assertEquals("committer should not have committed", 0, jeh.numCommitCompletedEvents);
    // set a fresh heartbeat and verify commit completes
    rmhh.setLastHeartbeatTime(clock.getTime());
    timeToWaitMs = 5000;
    while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not complete commit after RM hearbeat", 1, jeh.numCommitCompletedEvents);
    verify(committer, times(1)).commitJob(any(JobContext.class));
    //Clean up so we can try to commit again (Don't do this at home)
    cleanup();
    // try to commit again and verify it goes through since the heartbeat
    // is still fresh
    ceh.handle(new CommitterJobCommitEvent(null, null));
    timeToWaitMs = 5000;
    while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not commit", 2, jeh.numCommitCompletedEvents);
    verify(committer, times(2)).commitJob(any(JobContext.class));
    ceh.stop();
    dispatcher.stop();
}