Examples with Severity com.linkedin.drelephant.analysis.Severity used on opensource projects

Example 1 with Severity

use of com.linkedin.drelephant.analysis.Severity in project dr-elephant by linkedin.

the class Web method restWorkflowFromFlowId.

/**
 * This method returns the workflow response object based on the flow execution id
 * @param flowId The flow execution id for which the flow should be returned
 * @return Return the workflow detail based on the flow execution id
 *
 * response object:
 * <pre>
 * *{
 *    "workflows": {
 *    "id": "flowid",
 *        "username": "username",
 *        "starttime": 1471910835628,
 *        "finishtime": 1471911099238,
 *        "runtime": 263610,
 *        "waittime": 46234,
 *        "resourceused": 101382144,
 *        "resourcewasted": 15993417,
 *        "severity": "Moderate",
 *        "flowexecid": "flowexecid",
 *        "flowdefid": "flowdefid",
 *        "jobssummaries": [
 *          {
 *            "id": "jobid",
 *            "jobname": "jobname",
 *            "jobtype": "Pig",
 *            "username": "username",
 *            "starttime": 1471910835628,
 *            "finishtime": 1471911099238,
 *            "runtime": 263610,
 *            "waittime": 46234,
 *            "resourceused": 101382144,
 *            "resourcewasted": 15993417,
 *            "severity": "Moderate",
 *            "tasksseverity": [
 *              {
 *                "severity": "Moderate",
 *                "count": 1
 *              }
 *             ]
 *        }
 *      ],
 *        "jobsseverity": [
 *          {
 *            "severity": "Moderate",
 *            "count": 1
 *          }
 *      ]
 *  }
 *}
 * </pre>
 */
public static Result restWorkflowFromFlowId(String flowId) {
    if (flowId == null || flowId.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.WORKFLOWS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    JsonArray jobSeverityArray = new JsonArray();
    JsonArray jobSummaryArray = new JsonArray();
    JsonObject data = new JsonObject();
    String flowExecId = flowId;
    String username = "";
    long totalFlowResourceUsed = 0;
    long totalFlowResourceWasted = 0;
    long totalFlowRuntime = 0;
    long totalFlowDelay = 0;
    Severity flowSeverity = Severity.NONE;
    long flowStartTime = Long.MAX_VALUE;
    long flowEndTime = 0;
    String flowDefinitionId = "";
    Map<Severity, Long> jobSeverityCount = new HashMap<Severity, Long>();
    String wfQueueName = "";
    String wfSchedulerName = "";
    List<AppResult> results = getRestFlowResultsFromFlowExecutionId(flowId);
    if (results.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.WORKFLOWS, data);
        return notFound(new Gson().toJson(parent));
    }
    Map<IdUrlPair, List<AppResult>> jobExecIdToJobsMap = ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.JOB_EXECUTION_ID);
    for (IdUrlPair jobDefPair : jobExecIdToJobsMap.keySet()) {
        long totalJobMemoryUsed = 0;
        long totalJobMemoryWasted = 0;
        long totalJobDelay = 0;
        long totalJobRuntime = 0;
        long jobStartTime = Long.MAX_VALUE;
        long jobEndTime = 0;
        Severity jobSeverity = Severity.NONE;
        String jobType = null;
        String jobId = jobDefPair.getId();
        String jobName = "";
        String queueName = "";
        String schedulerName = "";
        Map<Severity, Long> taskSeverityCount = new HashMap<Severity, Long>();
        for (AppResult task : jobExecIdToJobsMap.get(jobDefPair)) {
            totalJobMemoryUsed += task.resourceUsed;
            totalJobMemoryWasted += task.resourceWasted;
            username = task.username;
            jobType = task.jobType;
            jobName = task.jobName;
            flowDefinitionId = task.flowDefId;
            queueName = task.queueName;
            schedulerName = task.scheduler;
            if (task.startTime < jobStartTime) {
                jobStartTime = task.startTime;
            }
            if (task.finishTime > jobEndTime) {
                jobEndTime = task.finishTime;
            }
            if (task.severity.getValue() > jobSeverity.getValue()) {
                jobSeverity = task.severity;
            }
            if (taskSeverityCount.containsKey(task.severity)) {
                taskSeverityCount.put(task.severity, taskSeverityCount.get(task.severity) + 1L);
            } else {
                taskSeverityCount.put(task.severity, 1L);
            }
        }
        if (jobSeverityCount.containsKey(jobSeverity)) {
            jobSeverityCount.put(jobSeverity, jobSeverityCount.get(jobSeverity) + 1L);
        } else {
            jobSeverityCount.put(jobSeverity, 1L);
        }
        JsonArray taskSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(taskSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, taskSeverityCount.get(key));
            taskSeverity.add(severityObject);
        }
        wfQueueName = queueName;
        wfSchedulerName = schedulerName;
        totalJobDelay = Utils.getTotalWaittime(jobExecIdToJobsMap.get(jobDefPair));
        totalJobRuntime = Utils.getTotalRuntime(jobExecIdToJobsMap.get(jobDefPair));
        JsonObject jobObject = new JsonObject();
        jobObject.addProperty(JsonKeys.ID, jobId);
        jobObject.addProperty(JsonKeys.JOB_NAME, jobName);
        jobObject.addProperty(JsonKeys.JOB_TYPE, jobType);
        jobObject.addProperty(JsonKeys.USERNAME, username);
        jobObject.addProperty(JsonKeys.START_TIME, jobStartTime);
        jobObject.addProperty(JsonKeys.FINISH_TIME, jobEndTime);
        jobObject.addProperty(JsonKeys.RUNTIME, totalJobRuntime);
        jobObject.addProperty(JsonKeys.WAITTIME, totalJobDelay);
        jobObject.addProperty(JsonKeys.RESOURCE_USED, totalJobMemoryUsed);
        jobObject.addProperty(JsonKeys.RESOURCE_WASTED, totalJobMemoryWasted);
        jobObject.addProperty(JsonKeys.QUEUE, queueName);
        jobObject.addProperty(JsonKeys.SCHEDULER, schedulerName);
        jobObject.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
        jobObject.add(JsonKeys.TASKS_SEVERITY, taskSeverity);
        jobSummaryArray.add(jobObject);
        totalFlowResourceUsed += totalJobMemoryUsed;
        totalFlowResourceWasted += totalJobMemoryWasted;
        if (jobSeverity.getValue() > flowSeverity.getValue()) {
            flowSeverity = jobSeverity;
        }
        if (flowStartTime > jobStartTime) {
            flowStartTime = jobStartTime;
        }
        if (flowEndTime < jobEndTime) {
            flowEndTime = jobEndTime;
        }
    }
    // job map scope ends here
    List<Severity> keys = getSortedSeverityKeys(jobSeverityCount.keySet());
    for (Severity key : keys) {
        JsonObject severityObject = new JsonObject();
        severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
        severityObject.addProperty(JsonKeys.COUNT, jobSeverityCount.get(key));
        jobSeverityArray.add(severityObject);
    }
    totalFlowDelay = Utils.getTotalWaittime(results);
    totalFlowRuntime = Utils.getTotalRuntime(results);
    data.addProperty(JsonKeys.ID, flowExecId);
    data.addProperty(JsonKeys.USERNAME, username);
    data.addProperty(JsonKeys.START_TIME, flowStartTime);
    data.addProperty(JsonKeys.FINISH_TIME, flowEndTime);
    data.addProperty(JsonKeys.RUNTIME, totalFlowRuntime);
    data.addProperty(JsonKeys.WAITTIME, totalFlowDelay);
    data.addProperty(JsonKeys.RESOURCE_USED, totalFlowResourceUsed);
    data.addProperty(JsonKeys.RESOURCE_WASTED, totalFlowResourceWasted);
    data.addProperty(JsonKeys.SEVERITY, flowSeverity.getText());
    data.addProperty(JsonKeys.FLOW_EXEC_ID, flowExecId);
    data.addProperty(JsonKeys.FLOW_DEF_ID, flowDefinitionId);
    data.addProperty(JsonKeys.QUEUE, wfQueueName);
    data.addProperty(JsonKeys.SCHEDULER, wfSchedulerName);
    data.add(JsonKeys.JOBSSUMMARIES, jobSummaryArray);
    data.add(JsonKeys.JOBS_SEVERITY, jobSeverityArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.WORKFLOWS, data);
    return ok(new Gson().toJson(parent));
}

Example 2 with Severity

use of com.linkedin.drelephant.analysis.Severity in project dr-elephant by linkedin.

the class Web method restSearchOptions.

/**
 * This returns the rest search options which are filled in the forms for the search page.
 * @return Returns the json object which should be filled in the search form.
 * return object:
 * <pre>
 *  *{
 *  "search-options": {
 *    "jobcategory": [
 *      {
 *        "name": "SPARK",
 *        "jobtypes": [
 *          {
 *            "name": "Spark"
 *          }
 *        ],
 *        "heuristics": [
 *          {
 *            "name": "Spark Configuration Best Practice"
 *          },
 *          {
 *            "name": "Spark Memory Limit"
 *          },
 *          {
 *            "name": "Spark Stage Runtime"
 *          },
 *          {
 *            "name": "Spark Job Runtime"
 *          },
 *          {
 *            "name": "Spark Executor Load Balance"
 *          },
 *          {
 *            "name": "Spark Event Log Limit"
 *          }
 *        ]
 *      },
 *      {
 *        "name": "MAPREDUCE",
 *        "jobtypes": [
 *          {
 *            "name": "Pig"
 *          },
 *          {
 *            "name": "Hive"
 *          },
 *          {
 *            "name": "Cascading"
 *          },
 *          {
 *            "name": "Voldemort"
 *          },
 *          {
 *            "name": "Kafka"
 *          },
 *          {
 *            "name": "HadoopJava"
 *          }
 *        ],
 *        "heuristics": [
 *          {
 *            "name": "Mapper Data Skew"
 *          },
 *          {
 *            "name": "Mapper GC"
 *          },
 *          {
 *            "name": "Mapper Time"
 *          },
 *          {
 *            "name": "Mapper Speed"
 *          },
 *          {
 *            "name": "Mapper Spill"
 *          },
 *          {
 *            "name": "Mapper Memory"
 *          },
 *          {
 *            "name": "Reducer Data Skew"
 *          },
 *          {
 *            "name": "Reducer GC"
 *          },
 *          {
 *            "name": "Reducer Time"
 *          },
 *          {
 *            "name": "Reducer Memory"
 *          },
 *          {
 *            "name": "Shuffle & Sort"
 *          },
 *          {
 *            "name": "Exception"
 *          }
 *        ]
 *      }
 *    ],
 *    "severities": [
 *      {
 *        "name": "Critical",
 *        "value": 4
 *      },
 *      {
 *        "name": "Severe",
 *        "value": 3
 *      },
 *      {
 *        "name": "Moderate",
 *        "value": 2
 *      },
 *      {
 *        "name": "Low",
 *        "value": 1
 *      },
 *      {
 *        "name": "None",
 *        "value": 0
 *      }
 *    ],
 *    "id": "search"
 *  }
 *}
 * </pre>
 */
public static Result restSearchOptions() {
    JsonObject searchOptions = new JsonObject();
    JsonArray jobCategory = new JsonArray();
    JsonArray severities = new JsonArray();
    Map<ApplicationType, List<JobType>> applicationTypeListMap = ElephantContext.instance().getAppTypeToJobTypes();
    for (ApplicationType key : applicationTypeListMap.keySet()) {
        JsonObject applicationType = new JsonObject();
        JsonArray jobTypes = new JsonArray();
        JsonArray heuristics = new JsonArray();
        for (JobType jobtype : applicationTypeListMap.get(key)) {
            JsonObject jobTypeNode = new JsonObject();
            jobTypeNode.addProperty(JsonKeys.NAME, jobtype.getName());
            jobTypes.add(jobTypeNode);
        }
        for (Heuristic heuristic : ElephantContext.instance().getHeuristicsForApplicationType(key)) {
            JsonObject heuristicNode = new JsonObject();
            heuristicNode.addProperty(JsonKeys.NAME, heuristic.getHeuristicConfData().getHeuristicName());
            heuristics.add(heuristicNode);
        }
        applicationType.addProperty(JsonKeys.NAME, key.getName());
        applicationType.add(JsonKeys.JOB_TYPES, jobTypes);
        applicationType.add(JsonKeys.HEURISTICS, heuristics);
        jobCategory.add(applicationType);
    }
    for (Severity severity : Severity.values()) {
        JsonObject severityObject = new JsonObject();
        severityObject.addProperty(JsonKeys.NAME, severity.getText());
        severityObject.addProperty(JsonKeys.VALUE, severity.getValue());
        severities.add(severityObject);
    }
    searchOptions.add(JsonKeys.JOB_CATEGORY, jobCategory);
    searchOptions.add(JsonKeys.SEVERITIES, severities);
    searchOptions.addProperty(JsonKeys.ID, "search");
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.SEARCH_OPTS, searchOptions);
    return ok(new Gson().toJson(parent));
}

Example 3 with Severity

use of com.linkedin.drelephant.analysis.Severity in project dr-elephant by linkedin.

the class Web method restWorkflowSummariesForUser.

/**
 * This method returns the workflow-summaries json response
 * @param username The username for which workflow-summaries must be returned
 * @return The json response of the workflow-summaries for the given user
 * Response data:
 * <pre>
 *{
 *  "workflow-summaries": [
 *  {
 *    "id": "http://workflow-id",
 *      "username": "search",
 *      "starttime": 1468818098875,
 *      "finishtime": 1468819946683,
 *      "runtime": 1855532,
 *      "waittime": 365368,
 *      "resourceused": 3306438656,
 *      "resourcewasted": 516978829,
 *      "severity": "Severe",
 *      "jobsseverity": [
 *    {
 *      "severity": "Severe",
 *        "count": 26
 *    },
 *    {
 *      "severity": "Moderate",
 *        "count": 3
 *    },
 *    {
 *      "severity": "Low",
 *        "count": 1
 *    },
 *    {
 *      "severity": "None",
 *        "count": 16
 *    }
 *    ]
 *  }
 *  ]
 *}
 * </pre>
 */
public static Result restWorkflowSummariesForUser(String username) {
    JsonArray workflowSummaryArray = new JsonArray();
    List<AppResult> results = null;
    if (username == null || username.isEmpty()) {
        results = getSchedulerApplications(MAX_APPLICATIONS_IN_WORKFLOW);
    } else {
        results = getSchedulerApplications(username, MAX_APPLICATIONS_IN_WORKFLOW);
    }
    Map<IdUrlPair, List<AppResult>> flowExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID), results.size(), MAX_FLOW_LIMIT);
    List<IdUrlPair> keyList = new ArrayList<IdUrlPair>(flowExecIdToJobsMap.keySet());
    for (IdUrlPair flowExecPair : keyList) {
        List<AppResult> mrJobsList = Lists.reverse(flowExecIdToJobsMap.get(flowExecPair));
        Map<IdUrlPair, List<AppResult>> jobDefIdToJobsMap = ControllerUtil.groupJobs(mrJobsList, ControllerUtil.GroupBy.JOB_EXECUTION_ID);
        Map<Severity, Long> jobSeverityCount = new HashMap<Severity, Long>();
        long totalFlowMemoryUsed = 0;
        long totalFlowMemoryWasted = 0;
        long totalFlowDelay = 0;
        long totalFlowRuntime = 0;
        Severity flowSeverity = Severity.NONE;
        for (IdUrlPair jobDefPair : jobDefIdToJobsMap.keySet()) {
            Severity jobseverity = Severity.NONE;
            long totalJobMemoryUsed = 0;
            long totalJobMemoryWasted = 0;
            for (AppResult job : jobDefIdToJobsMap.get(jobDefPair)) {
                totalJobMemoryUsed += job.resourceUsed;
                totalJobMemoryWasted += job.resourceWasted;
                if (job.severity.getValue() > jobseverity.getValue()) {
                    jobseverity = job.severity;
                }
            }
            if (jobSeverityCount.containsKey(jobseverity)) {
                jobSeverityCount.put(jobseverity, jobSeverityCount.get(jobseverity) + 1);
            } else {
                jobSeverityCount.put(jobseverity, 1L);
            }
            if (jobseverity.getValue() > flowSeverity.getValue()) {
                flowSeverity = jobseverity;
            }
            totalFlowMemoryUsed += totalJobMemoryUsed;
            totalFlowMemoryWasted += totalJobMemoryWasted;
        }
        totalFlowDelay = Utils.getTotalWaittime(flowExecIdToJobsMap.get(flowExecPair));
        totalFlowRuntime = Utils.getTotalRuntime(flowExecIdToJobsMap.get(flowExecPair));
        JsonArray jobSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(jobSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, jobSeverityCount.get(key));
            jobSeverity.add(severityObject);
        }
        // Execution record
        JsonObject dataset = new JsonObject();
        dataset.addProperty(JsonKeys.ID, mrJobsList.get(0).flowExecId);
        dataset.addProperty(JsonKeys.USERNAME, mrJobsList.get(0).username);
        dataset.addProperty(JsonKeys.START_TIME, mrJobsList.get(0).startTime);
        dataset.addProperty(JsonKeys.FINISH_TIME, mrJobsList.get(mrJobsList.size() - 1).finishTime);
        dataset.addProperty(JsonKeys.RUNTIME, totalFlowRuntime);
        dataset.addProperty(JsonKeys.WAITTIME, totalFlowDelay);
        dataset.addProperty(JsonKeys.RESOURCE_USED, totalFlowMemoryUsed);
        dataset.addProperty(JsonKeys.RESOURCE_WASTED, totalFlowMemoryWasted);
        dataset.addProperty(JsonKeys.QUEUE, mrJobsList.get(0).queueName);
        dataset.addProperty(JsonKeys.SEVERITY, flowSeverity.getText());
        dataset.addProperty(JsonKeys.SCHEDULER, mrJobsList.get(0).scheduler);
        dataset.addProperty(JsonKeys.FLOW_EXEC_ID, mrJobsList.get(0).flowExecId);
        dataset.addProperty(JsonKeys.FLOW_DEF_ID, mrJobsList.get(0).flowDefId);
        dataset.add(JsonKeys.JOBS_SEVERITY, jobSeverity);
        workflowSummaryArray.add(dataset);
    }
    JsonArray sortedWorkflowSummaryArray = getSortedJsonArrayByFinishTime(workflowSummaryArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.WORKFLOW_SUMMARIES, sortedWorkflowSummaryArray);
    return ok(new Gson().toJson(parent));
}

Example 4 with Severity

use of com.linkedin.drelephant.analysis.Severity in project dr-elephant by linkedin.

the class Web method restJobFromJobId.

/**
 * @param jobId
 * @return
 * <pre>
 **{
 *  "jobs": {
 *    "id": "jobid",
 *    "username": "username",
 *    "jobname": "jobname",
 *    "jobtype": "Pig",
 *    "starttime": 1471910835628,
 *    "finishtime": 1471911099238,
 *    "runtime": 263610,
 *    "waittime": 46234,
 *    "resourceused": 101382144,
 *    "resourcewasted": 15993417,
 *    "severity": "Moderate",
 *    "jobexecid": "jobexecid",
 *    "jobdefid": "jobdefid",
 *    "flowexecid": "flowexecid",
 *    "flowdefid": "flowdefid",
 *    "taskssummaries": [
 *      {
 *        "id": "application_id",
 *        "username": "username",
 *        "starttime": 1471910835628,
 *        "finishtime": 1471911099238,
 *        "runtime": 263610,
 *        "waittime": 46234,
 *        "resourceused": 101382144,
 *        "resourcewasted": 15993417,
 *        "severity": "Moderate",
 *        "heuristicsummary": [
 *          {
 *            "name": "Mapper Data Skew",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Mapper GC",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Mapper Time",
 *            "severity": "Moderate"
 *          },
 *          {
 *            "name": "Mapper Speed",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Mapper Spill",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Mapper Memory",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Reducer Data Skew",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Reducer GC",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Reducer Time",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Reducer Memory",
 *            "severity": "None"
 *          },
 *          {
 *            "name": "Shuffle & Sort",
 *            "severity": "Low"
 *          }
 *        ]
 *      }
 *    ],
 *    "tasksseverity": [
 *      {
 *        "severity": "Moderate",
 *        "count": 1
 *      }
 *    ]
 *  }
 *}
 *
 * </pre>
 */
public static Result restJobFromJobId(String jobid) {
    if (jobid == null || jobid.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.JOBS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    JsonArray taskSummaryArray = new JsonArray();
    String jobDefID = jobid;
    long jobResourceUsed = 0;
    long jobResourceWasted = 0;
    long jobRuntime = 0;
    long jobDelay = 0;
    Severity jobSeverity = Severity.NONE;
    long jobStartTime = Long.MAX_VALUE;
    long jobEndTime = 0;
    String username = "";
    String jobtype = "";
    String jobExecutionId = "";
    String jobDefinitionId = "";
    String flowExecutionId = "";
    String flowDefinitionId = "";
    String jobname = "";
    String queueName = "";
    String scheduler = "";
    List<AppResult> results = getRestJobResultsFromJobExecutionId(jobid);
    if (results.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.JOBS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    Map<Severity, Long> taskSeverityCount = new HashMap<Severity, Long>();
    for (AppResult task : results) {
        username = task.username;
        jobtype = task.jobType;
        jobname = task.jobName;
        jobExecutionId = task.jobExecId;
        jobDefinitionId = task.jobDefId;
        flowExecutionId = task.flowExecId;
        flowDefinitionId = task.flowDefId;
        queueName = task.queueName;
        scheduler = task.scheduler;
        JsonObject taskObject = new JsonObject();
        JsonArray heuristicsArray = new JsonArray();
        List<AppHeuristicResult> appHeuristicResult = task.yarnAppHeuristicResults;
        for (AppHeuristicResult heuristic : appHeuristicResult) {
            JsonObject heuristicObject = new JsonObject();
            heuristicObject.addProperty(JsonKeys.NAME, heuristic.heuristicName);
            heuristicObject.addProperty(JsonKeys.SEVERITY, heuristic.severity.getText());
            heuristicsArray.add(heuristicObject);
        }
        if (task.severity.getValue() > jobSeverity.getValue()) {
            jobSeverity = task.severity;
        }
        if (taskSeverityCount.containsKey(task.severity)) {
            taskSeverityCount.put(task.severity, taskSeverityCount.get(task.severity) + 1L);
        } else {
            taskSeverityCount.put(task.severity, 1L);
        }
        taskObject.addProperty(JsonKeys.ID, task.id);
        taskObject.addProperty(JsonKeys.USERNAME, task.username);
        taskObject.addProperty(JsonKeys.START_TIME, task.startTime);
        taskObject.addProperty(JsonKeys.FINISH_TIME, task.finishTime);
        taskObject.addProperty(JsonKeys.RUNTIME, task.finishTime - task.startTime);
        taskObject.addProperty(JsonKeys.WAITTIME, task.totalDelay);
        taskObject.addProperty(JsonKeys.RESOURCE_USED, task.resourceUsed);
        taskObject.addProperty(JsonKeys.RESOURCE_WASTED, task.resourceWasted);
        taskObject.addProperty(JsonKeys.SEVERITY, task.severity.getText());
        taskObject.addProperty(JsonKeys.QUEUE, task.queueName);
        taskObject.add(JsonKeys.HEURISTICS_SUMMARY, heuristicsArray);
        taskSummaryArray.add(taskObject);
        jobResourceUsed += task.resourceUsed;
        jobResourceWasted += task.resourceWasted;
        if (jobSeverity.getValue() < task.severity.getValue()) {
            jobSeverity = task.severity;
        }
        if (jobStartTime > task.startTime) {
            jobStartTime = task.startTime;
        }
        if (jobEndTime < task.finishTime) {
            jobEndTime = task.finishTime;
        }
    }
    JsonArray taskSeverity = new JsonArray();
    List<Severity> keys = getSortedSeverityKeys(taskSeverityCount.keySet());
    for (Severity key : keys) {
        JsonObject severityObject = new JsonObject();
        severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
        severityObject.addProperty(JsonKeys.COUNT, taskSeverityCount.get(key));
        taskSeverity.add(severityObject);
    }
    jobRuntime = Utils.getTotalRuntime(results);
    jobDelay = Utils.getTotalWaittime(results);
    JsonObject data = new JsonObject();
    data.addProperty(JsonKeys.ID, jobDefID);
    data.addProperty(JsonKeys.USERNAME, username);
    data.addProperty(JsonKeys.JOB_NAME, jobname);
    data.addProperty(JsonKeys.JOB_TYPE, jobtype);
    data.addProperty(JsonKeys.START_TIME, jobStartTime);
    data.addProperty(JsonKeys.FINISH_TIME, jobEndTime);
    data.addProperty(JsonKeys.RUNTIME, jobRuntime);
    data.addProperty(JsonKeys.WAITTIME, jobDelay);
    data.addProperty(JsonKeys.RESOURCE_USED, jobResourceUsed);
    data.addProperty(JsonKeys.RESOURCE_WASTED, jobResourceWasted);
    data.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
    data.addProperty(JsonKeys.JOB_EXEC_ID, jobExecutionId);
    data.addProperty(JsonKeys.JOB_DEF_ID, jobDefinitionId);
    data.addProperty(JsonKeys.FLOW_EXEC_ID, flowExecutionId);
    data.addProperty(JsonKeys.FLOW_DEF_ID, flowDefinitionId);
    data.addProperty(JsonKeys.QUEUE, queueName);
    data.addProperty(JsonKeys.SCHEDULER, scheduler);
    data.add(JsonKeys.TASKS_SUMMARIES, taskSummaryArray);
    data.add(JsonKeys.TASKS_SEVERITY, taskSeverity);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.JOBS, data);
    return ok(new Gson().toJson(parent));
}

Example 5 with Severity

use of com.linkedin.drelephant.analysis.Severity in project dr-elephant by linkedin.

the class MapperSpeedHeuristic method apply.

public HeuristicResult apply(TezApplicationData data) {
    if (!data.getSucceeded()) {
        return null;
    }
    TezTaskData[] tasks = data.getMapTaskData();
    List<Long> inputSizes = new ArrayList<Long>();
    List<Long> speeds = new ArrayList<Long>();
    List<Long> runtimesMs = new ArrayList<Long>();
    for (TezTaskData task : tasks) {
        if (task.isSampled()) {
            long inputBytes = 0;
            for (TezCounterData.CounterName counterName : _counterNames) {
                inputBytes += task.getCounters().get(counterName);
            }
            long runtimeMs = task.getTotalRunTimeMs();
            inputSizes.add(inputBytes);
            runtimesMs.add(runtimeMs);
            // Speed is records per second
            speeds.add((1000 * inputBytes) / (runtimeMs));
        }
    }
    long medianSpeed;
    long medianSize;
    long medianRuntimeMs;
    if (tasks.length != 0) {
        medianSpeed = Statistics.median(speeds);
        medianSize = Statistics.median(inputSizes);
        medianRuntimeMs = Statistics.median(runtimesMs);
    } else {
        medianSpeed = 0;
        medianSize = 0;
        medianRuntimeMs = 0;
    }
    Severity severity = getDiskSpeedSeverity(medianSpeed);
    // This reduces severity if task runtime is insignificant
    severity = Severity.min(severity, getRuntimeSeverity(medianRuntimeMs));
    HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length));
    result.addResultDetail("Number of tasks", Integer.toString(tasks.length));
    result.addResultDetail("Median task input ", FileUtils.byteCountToDisplaySize(medianSize));
    result.addResultDetail("Median task runtime", Statistics.readableTimespan(medianRuntimeMs));
    result.addResultDetail("Median task speed", FileUtils.byteCountToDisplaySize(medianSpeed) + "/s");
    return result;
}