Search in sources :

Example 1 with AMStartedEvent

use of org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent in project hadoop by apache.

the class MRAppMaster method readJustAMInfos.

private List<AMInfo> readJustAMInfos() {
    List<AMInfo> amInfos = new ArrayList<AMInfo>();
    FSDataInputStream inputStream = null;
    try {
        inputStream = getPreviousJobHistoryStream(getConfig(), appAttemptID);
        EventReader jobHistoryEventReader = new EventReader(inputStream);
        // All AMInfos are contiguous. Track when the first AMStartedEvent
        // appears.
        boolean amStartedEventsBegan = false;
        HistoryEvent event;
        while ((event = jobHistoryEventReader.getNextEvent()) != null) {
            if (event.getEventType() == EventType.AM_STARTED) {
                if (!amStartedEventsBegan) {
                    // First AMStartedEvent.
                    amStartedEventsBegan = true;
                }
                AMStartedEvent amStartedEvent = (AMStartedEvent) event;
                amInfos.add(MRBuilderUtils.newAMInfo(amStartedEvent.getAppAttemptId(), amStartedEvent.getStartTime(), amStartedEvent.getContainerId(), StringInterner.weakIntern(amStartedEvent.getNodeManagerHost()), amStartedEvent.getNodeManagerPort(), amStartedEvent.getNodeManagerHttpPort()));
            } else if (amStartedEventsBegan) {
                // No need to continue reading all the other events.
                break;
            }
        }
    } catch (IOException e) {
        LOG.warn("Could not parse the old history file. " + "Will not have old AMinfos ", e);
    } finally {
        if (inputStream != null) {
            IOUtils.closeQuietly(inputStream);
        }
    }
    return amInfos;
}
Also used : AMInfo(org.apache.hadoop.mapreduce.v2.api.records.AMInfo) AMStartedEvent(org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent) EventReader(org.apache.hadoop.mapreduce.jobhistory.EventReader) ArrayList(java.util.ArrayList) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) IOException(java.io.IOException) JobHistoryEvent(org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent) HistoryEvent(org.apache.hadoop.mapreduce.jobhistory.HistoryEvent)

Example 2 with AMStartedEvent

use of org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent in project hadoop by apache.

the class MRAppMaster method serviceStart.

@SuppressWarnings("unchecked")
@Override
protected void serviceStart() throws Exception {
    amInfos = new LinkedList<AMInfo>();
    completedTasksFromPreviousRun = new HashMap<TaskId, TaskInfo>();
    processRecovery();
    // Current an AMInfo for the current AM generation.
    AMInfo amInfo = MRBuilderUtils.newAMInfo(appAttemptID, startTime, containerID, nmHost, nmPort, nmHttpPort);
    // /////////////////// Create the job itself.
    job = createJob(getConfig(), forcedState, shutDownMessage);
    // Send out an MR AM inited event for all previous AMs.
    for (AMInfo info : amInfos) {
        dispatcher.getEventHandler().handle(new JobHistoryEvent(job.getID(), new AMStartedEvent(info.getAppAttemptId(), info.getStartTime(), info.getContainerId(), info.getNodeManagerHost(), info.getNodeManagerPort(), info.getNodeManagerHttpPort(), appSubmitTime)));
    }
    // Send out an MR AM inited event for this AM.
    dispatcher.getEventHandler().handle(new JobHistoryEvent(job.getID(), new AMStartedEvent(amInfo.getAppAttemptId(), amInfo.getStartTime(), amInfo.getContainerId(), amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort(), amInfo.getNodeManagerHttpPort(), this.forcedState == null ? null : this.forcedState.toString(), appSubmitTime)));
    amInfos.add(amInfo);
    // metrics system init is really init & start.
    // It's more test friendly to put it here.
    DefaultMetricsSystem.initialize("MRAppMaster");
    boolean initFailed = false;
    if (!errorHappenedShutDown) {
        // create a job event for job intialization
        JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
        // Send init to the job (this does NOT trigger job execution)
        // This is a synchronous call, not an event through dispatcher. We want
        // job-init to be done completely here.
        jobEventDispatcher.handle(initJobEvent);
        // If job is still not initialized, an error happened during
        // initialization. Must complete starting all of the services so failure
        // events can be processed.
        initFailed = (((JobImpl) job).getInternalState() != JobStateInternal.INITED);
        if (job.isUber()) {
            speculatorEventDispatcher.disableSpeculation();
            LOG.info("MRAppMaster uberizing job " + job.getID() + " in local container (\"uber-AM\") on node " + nmHost + ":" + nmPort + ".");
        } else {
            // send init to speculator only for non-uber jobs. 
            // This won't yet start as dispatcher isn't started yet.
            dispatcher.getEventHandler().handle(new SpeculatorEvent(job.getID(), clock.getTime()));
            LOG.info("MRAppMaster launching normal, non-uberized, multi-container " + "job " + job.getID() + ".");
        }
        // Start ClientService here, since it's not initialized if
        // errorHappenedShutDown is true
        clientService.start();
    }
    //start all the components
    super.serviceStart();
    // finally set the job classloader
    MRApps.setClassLoader(jobClassLoader, getConfig());
    // set job classloader if configured
    Limits.init(getConfig());
    if (initFailed) {
        JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
        jobEventDispatcher.handle(initFailedEvent);
    } else {
        // All components have started, start the job.
        startJobs();
    }
}
Also used : AMInfo(org.apache.hadoop.mapreduce.v2.api.records.AMInfo) TaskInfo(org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo) AMStartedEvent(org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent) TaskId(org.apache.hadoop.mapreduce.v2.api.records.TaskId) JobEvent(org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent) SpeculatorEvent(org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent) JobHistoryEvent(org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent)

Aggregations

AMStartedEvent (org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent)2 JobHistoryEvent (org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent)2 AMInfo (org.apache.hadoop.mapreduce.v2.api.records.AMInfo)2 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 FSDataInputStream (org.apache.hadoop.fs.FSDataInputStream)1 EventReader (org.apache.hadoop.mapreduce.jobhistory.EventReader)1 HistoryEvent (org.apache.hadoop.mapreduce.jobhistory.HistoryEvent)1 TaskInfo (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo)1 TaskId (org.apache.hadoop.mapreduce.v2.api.records.TaskId)1 JobEvent (org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent)1 SpeculatorEvent (org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent)1