Examples with TaskType org.apache.hadoop.mapreduce.v2.api.records.TaskType used on opensource projects

Example 21 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class StartEndTimesBase method thresholdRuntime.

@Override
public long thresholdRuntime(TaskId taskID) {
    JobId jobID = taskID.getJobId();
    Job job = context.getJob(jobID);
    TaskType type = taskID.getTaskType();
    DataStatistics statistics = dataStatisticsForTask(taskID);
    int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces();
    int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces();
    if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float) completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE) {
        return Long.MAX_VALUE;
    }
    long result = statistics == null ? Long.MAX_VALUE : (long) statistics.outlier(slowTaskRelativeTresholds.get(job));
    return result;
}

Example 22 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class TestAMWebServicesAttempts method verifyTaskAttemptGeneric.

public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype, String id, String state, String type, String rack, String nodeHttpAddress, String diagnostics, String assignedContainerId, long startTime, long finishTime, long elapsedTime, float progress) {
    TaskAttemptId attid = ta.getID();
    String attemptId = MRApps.toString(attid);
    WebServicesTestUtils.checkStringMatch("id", attemptId, id);
    WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
    WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(), state);
    WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
    WebServicesTestUtils.checkStringMatch("nodeHttpAddress", ta.getNodeHttpAddress(), nodeHttpAddress);
    String expectDiag = "";
    List<String> diagnosticsList = ta.getDiagnostics();
    if (diagnosticsList != null && !diagnostics.isEmpty()) {
        StringBuffer b = new StringBuffer();
        for (String diag : diagnosticsList) {
            b.append(diag);
        }
        expectDiag = b.toString();
    }
    WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag, diagnostics);
    WebServicesTestUtils.checkStringMatch("assignedContainerId", ta.getAssignedContainerID().toString(), assignedContainerId);
    assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
    assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
    assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
    assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}

Example 23 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class HsTaskPage method attemptsTableInit.

/**
   * @return The end of the JS map that is the jquery datatable config for the
   * attempts table. 
   */
private String attemptsTableInit() {
    TaskType type = null;
    String symbol = $(TASK_TYPE);
    if (!symbol.isEmpty()) {
        type = MRApps.taskType(symbol);
    } else {
        TaskId taskID = MRApps.toTaskID($(TASK_ID));
        type = taskID.getTaskType();
    }
    StringBuilder b = tableInit().append(", 'aaData': attemptsTableData").append(", bDeferRender: true").append(", bProcessing: true").append("\n,aoColumnDefs:[\n").append("\n{'aTargets': [ 4 ]").append(", 'bSearchable': false }").append("\n, {'sType':'natural', 'aTargets': [ 0 ]").append(", 'mRender': parseHadoopID }").append("\n, {'sType':'numeric', 'aTargets': [ 5, 6").append(type == TaskType.REDUCE ? ", 7, 8" : "").append(" ], 'mRender': renderHadoopDate }").append("\n, {'sType':'numeric', 'aTargets': [").append(type == TaskType.REDUCE ? "9, 10, 11, 12" : "7").append(" ], 'mRender': renderHadoopElapsedTime }]").append("\n, aaSorting: [[0, 'asc']]").append("}");
    return b.toString();
}

Example 24 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class HsTasksBlock method render.

/*
   * (non-Javadoc)
   * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
   */
@Override
protected void render(Block html) {
    if (app.getJob() == null) {
        html.h2($(TITLE));
        return;
    }
    TaskType type = null;
    String symbol = $(TASK_TYPE);
    if (!symbol.isEmpty()) {
        type = MRApps.taskType(symbol);
    }
    THEAD<TABLE<Hamlet>> thead;
    if (type != null)
        thead = html.table("#" + app.getJob().getID() + type).$class("dt-tasks").thead();
    else
        thead = html.table("#tasks").thead();
    //Create the spanning row
    int attemptColSpan = type == TaskType.REDUCE ? 8 : 3;
    thead.tr().th().$colspan(5).$class("ui-state-default")._("Task")._().th().$colspan(attemptColSpan).$class("ui-state-default")._("Successful Attempt")._()._();
    TR<THEAD<TABLE<Hamlet>>> theadRow = thead.tr().th("Name").th("State").th("Start Time").th("Finish Time").th("Elapsed Time").th(//Attempt
    "Start Time");
    if (type == TaskType.REDUCE) {
        //Attempt
        theadRow.th("Shuffle Finish Time");
        //Attempt
        theadRow.th("Merge Finish Time");
    }
    //Attempt
    theadRow.th("Finish Time");
    if (type == TaskType.REDUCE) {
        //Attempt
        theadRow.th("Elapsed Time Shuffle");
        //Attempt
        theadRow.th("Elapsed Time Merge");
        //Attempt
        theadRow.th("Elapsed Time Reduce");
    }
    //Attempt
    theadRow.th("Elapsed Time");
    TBODY<TABLE<Hamlet>> tbody = theadRow._()._().tbody();
    // Write all the data into a JavaScript array of arrays for JQuery
    // DataTables to display
    StringBuilder tasksTableData = new StringBuilder("[\n");
    for (Task task : app.getJob().getTasks().values()) {
        if (type != null && task.getType() != type) {
            continue;
        }
        TaskInfo info = new TaskInfo(task);
        String tid = info.getId();
        long startTime = info.getStartTime();
        long finishTime = info.getFinishTime();
        long elapsed = info.getElapsedTime();
        long attemptStartTime = -1;
        long shuffleFinishTime = -1;
        long sortFinishTime = -1;
        long attemptFinishTime = -1;
        long elapsedShuffleTime = -1;
        long elapsedSortTime = -1;
        ;
        long elapsedReduceTime = -1;
        long attemptElapsed = -1;
        TaskAttempt successful = info.getSuccessful();
        if (successful != null) {
            TaskAttemptInfo ta;
            if (type == TaskType.REDUCE) {
                ReduceTaskAttemptInfo rta = new ReduceTaskAttemptInfo(successful, type);
                shuffleFinishTime = rta.getShuffleFinishTime();
                sortFinishTime = rta.getMergeFinishTime();
                elapsedShuffleTime = rta.getElapsedShuffleTime();
                elapsedSortTime = rta.getElapsedMergeTime();
                elapsedReduceTime = rta.getElapsedReduceTime();
                ta = rta;
            } else {
                ta = new TaskAttemptInfo(successful, type, false);
            }
            attemptStartTime = ta.getStartTime();
            attemptFinishTime = ta.getFinishTime();
            attemptElapsed = ta.getElapsedTime();
        }
        tasksTableData.append("[\"").append("<a href='" + url("task", tid)).append("'>").append(tid).append("</a>\",\"").append(info.getState()).append("\",\"").append(startTime).append("\",\"").append(finishTime).append("\",\"").append(elapsed).append("\",\"").append(attemptStartTime).append("\",\"");
        if (type == TaskType.REDUCE) {
            tasksTableData.append(shuffleFinishTime).append("\",\"").append(sortFinishTime).append("\",\"");
        }
        tasksTableData.append(attemptFinishTime).append("\",\"");
        if (type == TaskType.REDUCE) {
            tasksTableData.append(elapsedShuffleTime).append("\",\"").append(elapsedSortTime).append("\",\"").append(elapsedReduceTime).append("\",\"");
        }
        tasksTableData.append(attemptElapsed).append("\"],\n");
    }
    //Remove the last comma and close off the array of arrays
    if (tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
        tasksTableData.delete(tasksTableData.length() - 2, tasksTableData.length() - 1);
    }
    tasksTableData.append("]");
    html.script().$type("text/javascript")._("var tasksTableData=" + tasksTableData)._();
    TR<TFOOT<TABLE<Hamlet>>> footRow = tbody._().tfoot().tr();
    footRow.th().input("search_init").$type(InputType.text).$name("task").$value("ID")._()._().th().input("search_init").$type(InputType.text).$name("state").$value("State")._()._().th().input("search_init").$type(InputType.text).$name("start_time").$value("Start Time")._()._().th().input("search_init").$type(InputType.text).$name("finish_time").$value("Finish Time")._()._().th().input("search_init").$type(InputType.text).$name("elapsed_time").$value("Elapsed Time")._()._().th().input("search_init").$type(InputType.text).$name("attempt_start_time").$value("Start Time")._()._();
    if (type == TaskType.REDUCE) {
        footRow.th().input("search_init").$type(InputType.text).$name("shuffle_time").$value("Shuffle Time")._()._();
        footRow.th().input("search_init").$type(InputType.text).$name("merge_time").$value("Merge Time")._()._();
    }
    footRow.th().input("search_init").$type(InputType.text).$name("attempt_finish").$value("Finish Time")._()._();
    if (type == TaskType.REDUCE) {
        footRow.th().input("search_init").$type(InputType.text).$name("elapsed_shuffle_time").$value("Elapsed Shuffle Time")._()._();
        footRow.th().input("search_init").$type(InputType.text).$name("elapsed_merge_time").$value("Elapsed Merge Time")._()._();
        footRow.th().input("search_init").$type(InputType.text).$name("elapsed_reduce_time").$value("Elapsed Reduce Time")._()._();
    }
    footRow.th().input("search_init").$type(InputType.text).$name("attempt_elapsed").$value("Elapsed Time")._()._();
    footRow._()._()._();
}

Example 25 with TaskType

use of org.apache.hadoop.mapreduce.v2.api.records.TaskType in project hadoop by apache.

the class HsTasksPage method tasksTableInit.

/**
   * @return the end of the JS map that is the jquery datatable configuration
   * for the tasks table.
   */
private String tasksTableInit() {
    TaskType type = null;
    String symbol = $(TASK_TYPE);
    if (!symbol.isEmpty()) {
        type = MRApps.taskType(symbol);
    }
    StringBuilder b = tableInit().append(", 'aaData': tasksTableData").append(", bDeferRender: true").append(", bProcessing: true").append("\n, aoColumnDefs: [\n").append("{'sType':'natural', 'aTargets': [ 0 ]").append(", 'mRender': parseHadoopID }").append(", {'sType':'numeric', 'aTargets': [ 4").append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7").append(" ], 'mRender': renderHadoopElapsedTime }").append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5").append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6").append(" ], 'mRender': renderHadoopDate }]").append("\n, aaSorting: [[0, 'asc']]").append("}");
    return b.toString();
}