Examples with JobHistoryEvent org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent used on opensource projects

Example 6 with JobHistoryEvent

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent in project hadoop by apache.

the class RMContainerAllocator method handleMapContainerRequest.

@SuppressWarnings({ "unchecked" })
private void handleMapContainerRequest(ContainerRequestEvent reqEvent) {
    assert (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP));
    Resource supportedMaxContainerCapability = getMaxContainerCapability();
    JobId jobId = getJob().getID();
    if (mapResourceRequest.equals(Resources.none())) {
        mapResourceRequest = reqEvent.getCapability();
        eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.MAP, mapResourceRequest.getMemorySize())));
        LOG.info("mapResourceRequest:" + mapResourceRequest);
    }
    boolean mapContainerRequestAccepted = true;
    if (mapResourceRequest.getMemorySize() > supportedMaxContainerCapability.getMemorySize() || mapResourceRequest.getVirtualCores() > supportedMaxContainerCapability.getVirtualCores()) {
        mapContainerRequestAccepted = false;
    }
    if (mapContainerRequestAccepted) {
        // set the resources
        reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize());
        reqEvent.getCapability().setVirtualCores(mapResourceRequest.getVirtualCores());
        //maps are immediately scheduled
        scheduledRequests.addMap(reqEvent);
    } else {
        String diagMsg = "The required MAP capability is more than the " + "supported max container capability in the cluster. Killing" + " the Job. mapResourceRequest: " + mapResourceRequest + " maxContainerCapability:" + supportedMaxContainerCapability;
        LOG.info(diagMsg);
        eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
        eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    }
}

Example 7 with JobHistoryEvent

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent in project hadoop by apache.

the class RMContainerAllocator method handleReduceContainerRequest.

@SuppressWarnings({ "unchecked" })
private void handleReduceContainerRequest(ContainerRequestEvent reqEvent) {
    assert (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.REDUCE));
    Resource supportedMaxContainerCapability = getMaxContainerCapability();
    JobId jobId = getJob().getID();
    if (reduceResourceRequest.equals(Resources.none())) {
        reduceResourceRequest = reqEvent.getCapability();
        eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.REDUCE, reduceResourceRequest.getMemorySize())));
        LOG.info("reduceResourceRequest:" + reduceResourceRequest);
    }
    boolean reduceContainerRequestAccepted = true;
    if (reduceResourceRequest.getMemorySize() > supportedMaxContainerCapability.getMemorySize() || reduceResourceRequest.getVirtualCores() > supportedMaxContainerCapability.getVirtualCores()) {
        reduceContainerRequestAccepted = false;
    }
    if (reduceContainerRequestAccepted) {
        // set the resources
        reqEvent.getCapability().setVirtualCores(reduceResourceRequest.getVirtualCores());
        reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize());
        if (reqEvent.getEarlierAttemptFailed()) {
            //previously failed reducers are added to the front for fail fast
            pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE, reduceNodeLabelExpression));
        } else {
            //reduces are added to pending queue and are slowly ramped up
            pendingReduces.add(new ContainerRequest(reqEvent, PRIORITY_REDUCE, reduceNodeLabelExpression));
        }
    } else {
        String diagMsg = "REDUCE capability required is more than the " + "supported max container capability in the cluster. Killing" + " the Job. reduceResourceRequest: " + reduceResourceRequest + " maxContainerCapability:" + supportedMaxContainerCapability;
        LOG.info(diagMsg);
        eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
        eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    }
}

Example 8 with JobHistoryEvent

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent in project hadoop by apache.

the class JobImpl method unsuccessfulFinish.

private void unsuccessfulFinish(JobStateInternal finalState) {
    if (finishTime == 0)
        setFinishTime();
    cleanupProgress = 1.0f;
    JobUnsuccessfulCompletionEvent unsuccessfulJobEvent = new JobUnsuccessfulCompletionEvent(oldJobId, finishTime, succeededMapTaskCount, succeededReduceTaskCount, finalState.toString(), diagnostics);
    eventHandler.handle(new JobHistoryEvent(jobId, unsuccessfulJobEvent));
    finished(finalState);
}

Example 9 with JobHistoryEvent

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent in project hadoop by apache.

the class JobImpl method setQueueName.

@Override
public void setQueueName(String queueName) {
    this.queueName = queueName;
    JobQueueChangeEvent jqce = new JobQueueChangeEvent(oldJobId, queueName);
    eventHandler.handle(new JobHistoryEvent(jobId, jqce));
}

Example 10 with JobHistoryEvent

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent in project hadoop by apache.

the class JobImpl method logJobHistoryFinishedEvent.

void logJobHistoryFinishedEvent() {
    this.setFinishTime();
    JobFinishedEvent jfe = createJobFinishedEvent(this);
    LOG.info("Calling handler for JobFinishedEvent ");
    this.getEventHandler().handle(new JobHistoryEvent(this.jobId, jfe));
}