Examples with TaskAttemptId org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId used on opensource projects

Example 71 with TaskAttemptId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId in project hadoop by apache.

the class TestAMWebServicesAttempt method testGetTaskAttemptIdState.

@Test
public void testGetTaskAttemptIdState() throws Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        for (Task task : jobsMap.get(id).getTasks().values()) {
            String tid = MRApps.toString(task.getID());
            for (TaskAttempt att : task.getAttempts().values()) {
                TaskAttemptId attemptid = att.getID();
                String attid = MRApps.toString(attemptid);
                ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid).path("attempts").path(attid).path("state").queryParam("user.name", webserviceUserName).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
                assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
                JSONObject json = response.getEntity(JSONObject.class);
                assertEquals("incorrect number of elements", 1, json.length());
                assertEquals(att.getState().toString(), json.get("state"));
            }
        }
    }
}

Example 72 with TaskAttemptId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId in project hadoop by apache.

the class TestAMWebServicesAttempts method testTaskAttemptIdXML.

@Test
public void testTaskAttemptIdXML() throws JSONException, Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        for (Task task : jobsMap.get(id).getTasks().values()) {
            String tid = MRApps.toString(task.getID());
            for (TaskAttempt att : task.getAttempts().values()) {
                TaskAttemptId attemptid = att.getID();
                String attid = MRApps.toString(attemptid);
                ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid).path("attempts").path(attid).accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
                assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
                String xml = response.getEntity(String.class);
                DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
                DocumentBuilder db = dbf.newDocumentBuilder();
                InputSource is = new InputSource();
                is.setCharacterStream(new StringReader(xml));
                Document dom = db.parse(is);
                NodeList nodes = dom.getElementsByTagName("taskAttempt");
                for (int i = 0; i < nodes.getLength(); i++) {
                    Element element = (Element) nodes.item(i);
                    verifyAMTaskAttemptXML(element, att, task.getType());
                }
            }
        }
    }
}

Example 73 with TaskAttemptId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId in project tez by apache.

the class TezTypeConverters method toYarn.

public static TaskAttemptId toYarn(TezTaskAttemptID taskAttemptId) {
    TaskAttemptID mrTaskAttemptId = IDConverter.toMRTaskAttemptId(taskAttemptId);
    TaskAttemptId mrv2TaskAttemptId = TypeConverter.toYarn(mrTaskAttemptId);
    return mrv2TaskAttemptId;
}

Example 74 with TaskAttemptId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId in project hadoop by apache.

the class TaskAttemptListenerImpl method canCommit.

/**
   * Child checking whether it can commit.
   * 
   * <br>
   * Commit is a two-phased protocol. First the attempt informs the
   * ApplicationMaster that it is
   * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
   * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
   * a legacy from the centralized commit protocol handling by the JobTracker.
   */
@Override
public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException {
    LOG.info("Commit go/no-go request from " + taskAttemptID.toString());
    // An attempt is asking if it can commit its output. This can be decided
    // only by the task which is managing the multiple attempts. So redirect the
    // request there.
    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID);
    taskHeartbeatHandler.progressing(attemptID);
    // tell task to retry later if AM has not heard from RM within the commit
    // window to help avoid double-committing in a split-brain situation
    long now = context.getClock().getTime();
    if (now - rmHeartbeatHandler.getLastHeartbeatTime() > commitWindowMs) {
        return false;
    }
    Job job = context.getJob(attemptID.getTaskId().getJobId());
    Task task = job.getTask(attemptID.getTaskId());
    return task.canCommit(attemptID);
}

Example 75 with TaskAttemptId

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId in project hadoop by apache.

the class TaskAttemptListenerImpl method statusUpdate.

@Override
public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus) throws IOException, InterruptedException {
    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID = TypeConverter.toYarn(taskAttemptID);
    AMFeedback feedback = new AMFeedback();
    feedback.setTaskFound(true);
    // Propagating preemption to the task if TASK_PREEMPTION is enabled
    if (getConfig().getBoolean(MRJobConfig.TASK_PREEMPTION, false) && preemptionPolicy.isPreempted(yarnAttemptID)) {
        feedback.setPreemption(true);
        LOG.info("Setting preemption bit for task: " + yarnAttemptID + " of type " + yarnAttemptID.getTaskId().getTaskType());
    }
    if (taskStatus == null) {
        //We are using statusUpdate only as a simple ping
        if (LOG.isDebugEnabled()) {
            LOG.debug("Ping from " + taskAttemptID.toString());
        }
        return feedback;
    }
    // if we are here there is an actual status update to be processed
    taskHeartbeatHandler.progressing(yarnAttemptID);
    TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
    taskAttemptStatus.id = yarnAttemptID;
    // Task sends the updated progress to the TT.
    taskAttemptStatus.progress = taskStatus.getProgress();
    LOG.info("Progress of TaskAttempt " + taskAttemptID + " is : " + taskStatus.getProgress());
    // Task sends the updated state-string to the TT.
    taskAttemptStatus.stateString = taskStatus.getStateString();
    // Task sends the updated phase to the TT.
    taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
    // Counters are updated by the task. Convert counters into new format as
    // that is the primary storage format inside the AM to avoid multiple
    // conversions and unnecessary heap usage.
    taskAttemptStatus.counters = new org.apache.hadoop.mapreduce.Counters(taskStatus.getCounters());
    // Map Finish time set by the task (map only)
    if (taskStatus.getIsMap() && taskStatus.getMapFinishTime() != 0) {
        taskAttemptStatus.mapFinishTime = taskStatus.getMapFinishTime();
    }
    // Shuffle Finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getShuffleFinishTime() != 0) {
        taskAttemptStatus.shuffleFinishTime = taskStatus.getShuffleFinishTime();
    }
    // Sort finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getSortFinishTime() != 0) {
        taskAttemptStatus.sortFinishTime = taskStatus.getSortFinishTime();
    }
    //set the fetch failures
    if (taskStatus.getFetchFailedMaps() != null && taskStatus.getFetchFailedMaps().size() > 0) {
        taskAttemptStatus.fetchFailedMaps = new ArrayList<org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId>();
        for (TaskAttemptID failedMapId : taskStatus.getFetchFailedMaps()) {
            taskAttemptStatus.fetchFailedMaps.add(TypeConverter.toYarn(failedMapId));
        }
    }
    // Task sends the information about the nextRecordRange to the TT
    //    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
    //    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
    //    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
    //    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
    //    // This was used by TT to do counter updates only once every minute. So this
    //    // isn't ever changed by the Task itself.
    //    taskStatus.getIncludeCounters();
    context.getEventHandler().handle(new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id, taskAttemptStatus));
    return feedback;
}