Examples with JobCounterUpdateEvent org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent used on opensource projects

Example 6 with JobCounterUpdateEvent

use of org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent in project hadoop by apache.

the class TaskAttemptImpl method createJobCounterUpdateEventTAKilled.

private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
    if (taskType == TaskType.MAP) {
        jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
    } else {
        jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
    }
    if (!taskAlreadyCompleted) {
        updateMillisCounters(jce, taskAttempt);
    }
    return jce;
}

Example 7 with JobCounterUpdateEvent

use of org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent in project hadoop by apache.

the class LocalContainerAllocator method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(ContainerAllocatorEvent event) {
    if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
        LOG.info("Processing the event " + event.toString());
        // Assign the same container ID as the AM
        ContainerId cID = ContainerId.newContainerId(getContext().getApplicationAttemptId(), this.containerId.getContainerId());
        Container container = recordFactory.newRecordInstance(Container.class);
        container.setId(cID);
        NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort);
        container.setResource(Resource.newInstance(0, 0));
        container.setNodeId(nodeId);
        container.setContainerToken(null);
        container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
        if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
            JobCounterUpdateEvent jce = new JobCounterUpdateEvent(event.getAttemptID().getTaskId().getJobId());
            // TODO Setting OTHER_LOCAL_MAP for now.
            jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
            eventHandler.handle(jce);
        }
        eventHandler.handle(new TaskAttemptContainerAssignedEvent(event.getAttemptID(), container, applicationACLs));
    }
}

Example 8 with JobCounterUpdateEvent

use of org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent in project hadoop by apache.

the class CheckpointAMPreemptionPolicy method updateCheckpointCounters.

@SuppressWarnings({ "unchecked" })
private void updateCheckpointCounters(TaskId taskId, TaskCheckpointID cid) {
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINTS, 1);
    eventHandler.handle(jce);
    jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINT_BYTES, cid.getCheckpointBytes());
    eventHandler.handle(jce);
    jce = new JobCounterUpdateEvent(taskId.getJobId());
    jce.addCounterUpdate(JobCounter.CHECKPOINT_TIME, cid.getCheckpointTime());
    eventHandler.handle(jce);
}

Example 9 with JobCounterUpdateEvent

use of org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent in project hadoop by apache.

the class CheckpointAMPreemptionPolicy method updatePreemptionCounters.

@SuppressWarnings({ "unchecked" })
private void updatePreemptionCounters(TaskAttemptId yarnAttemptID) {
    if (!countedPreemptions.contains(yarnAttemptID)) {
        countedPreemptions.add(yarnAttemptID);
        JobCounterUpdateEvent jce = new JobCounterUpdateEvent(yarnAttemptID.getTaskId().getJobId());
        jce.addCounterUpdate(JobCounter.TASKS_REQ_PREEMPT, 1);
        eventHandler.handle(jce);
    }
}

Example 10 with JobCounterUpdateEvent

use of org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent in project hadoop by apache.

the class TestRecovery method recoveryChecker.

private void recoveryChecker(MapTaskImpl checkTask, TaskState finalState, Map<TaskAttemptID, TaskAttemptState> finalAttemptStates, ArgumentCaptor<Event> arg, List<EventType> expectedJobHistoryEvents, long expectedMapLaunches, long expectedFailedMaps) {
    assertEquals("Final State of Task", finalState, checkTask.getState());
    Map<TaskAttemptId, TaskAttempt> recoveredAttempts = checkTask.getAttempts();
    assertEquals("Expected Number of Task Attempts", finalAttemptStates.size(), recoveredAttempts.size());
    for (TaskAttemptID taID : finalAttemptStates.keySet()) {
        assertEquals("Expected Task Attempt State", finalAttemptStates.get(taID), recoveredAttempts.get(TypeConverter.toYarn(taID)).getState());
    }
    Iterator<Event> ie = arg.getAllValues().iterator();
    int eventNum = 0;
    long totalLaunchedMaps = 0;
    long totalFailedMaps = 0;
    boolean jobTaskEventReceived = false;
    while (ie.hasNext()) {
        Object current = ie.next();
        ++eventNum;
        LOG.info(eventNum + " " + current.getClass().getName());
        if (current instanceof JobHistoryEvent) {
            JobHistoryEvent jhe = (JobHistoryEvent) current;
            LOG.info(expectedJobHistoryEvents.get(0).toString() + " " + jhe.getHistoryEvent().getEventType().toString() + " " + jhe.getJobID());
            assertEquals(expectedJobHistoryEvents.get(0), jhe.getHistoryEvent().getEventType());
            expectedJobHistoryEvents.remove(0);
        } else if (current instanceof JobCounterUpdateEvent) {
            JobCounterUpdateEvent jcue = (JobCounterUpdateEvent) current;
            boolean containsUpdates = jcue.getCounterUpdates().size() > 0;
            // TaskAttempt recovery. Check that first.
            if (containsUpdates) {
                LOG.info("JobCounterUpdateEvent " + jcue.getCounterUpdates().get(0).getCounterKey() + " " + jcue.getCounterUpdates().get(0).getIncrementValue());
                if (jcue.getCounterUpdates().get(0).getCounterKey() == JobCounter.NUM_FAILED_MAPS) {
                    totalFailedMaps += jcue.getCounterUpdates().get(0).getIncrementValue();
                } else if (jcue.getCounterUpdates().get(0).getCounterKey() == JobCounter.TOTAL_LAUNCHED_MAPS) {
                    totalLaunchedMaps += jcue.getCounterUpdates().get(0).getIncrementValue();
                }
            }
        } else if (current instanceof JobTaskEvent) {
            JobTaskEvent jte = (JobTaskEvent) current;
            assertEquals(jte.getState(), finalState);
            jobTaskEventReceived = true;
        }
    }
    assertTrue(jobTaskEventReceived || (finalState == TaskState.RUNNING));
    assertEquals("Did not process all expected JobHistoryEvents", 0, expectedJobHistoryEvents.size());
    assertEquals("Expected Map Launches", expectedMapLaunches, totalLaunchedMaps);
    assertEquals("Expected Failed Maps", expectedFailedMaps, totalFailedMaps);
}