Examples with AppHeuristicResultDetails models.AppHeuristicResultDetails used on opensource projects

Example 1 with AppHeuristicResultDetails

use of models.AppHeuristicResultDetails in project dr-elephant by linkedin.

the class Web method restApplicationFromApplicationId.

/**
 * @param applicationId
 * @return
 * <pre>
 *  {
 *  "applications": {
 *    "id": "application_id",
 *    "username": "username",
 *    "jobtype": "Pig",
 *    "mapreducejobname": "mapreducejobname",
 *    "starttime": 1471910835628,
 *    "finishtime": 1471911099238,
 *    "runtime": 263610,
 *    "waittime": 46234,
 *    "resourceused": 101382144,
 *    "resourcewasted": 15993417,
 *    "severity": "Moderate",
 *    "trackingurl": "jobtracker_address",
 *    "jobexecid": "jobexecutionid",
 *    "jobdefid": "jobdefinitionid",
 *    "flowexeid": "flowexecutionid",
 *    "flowdefid": "flowdefinitionid",
 *    "yarnappheuristicresults": [
 *      {
 *        "name": "Mapper Data Skew",
 *        "severity": "None",
 *        "details": [
 *          {
 *            "name": "Group A",
 *            "value": "236 tasks @ 506 MB avg"
 *          },
 *          {
 *            "name": "Group B",
 *            "value": "234 tasks @ 507 MB avg"
 *          },
 *          {
 *            "name": "Number of tasks",
 *            "value": "470"
 *          }
 *        ]
 *      },
 *      {
 *        "name": "Mapper GC",
 *        "severity": "None",
 *        "details": [
 *          {
 *            "name": "Avg task CPU time (ms)",
 *            "value": "111717"
 *          },
 *          {
 *            "name": "Avg task GC time (ms)",
 *            "value": "3197"
 *          },
 *          {
 *            "name": "Avg task runtime (ms)",
 *            "value": "105633"
 *          },
 *          {
 *            "name": "Number of tasks",
 *            "value": "470"
 *          },
 *          {
 *            "name": "Task GC\/CPU ratio",
 *            "value": "0.028616951762041588"
 *          }
 *        ]
 *      }..
 *    ]
 *  }
 *}
 * </pre>
 */
public static Result restApplicationFromApplicationId(String applicationid) {
    if (applicationid == null || applicationid.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.APPLICATIONS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    if (applicationid.startsWith("job")) {
        applicationid = applicationid.replaceAll("job", "application");
    }
    JsonObject applicationObject = new JsonObject();
    JsonArray heuristicsArray = new JsonArray();
    AppResult result = getAppResultFromApplicationId(applicationid);
    if (result == null) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.APPLICATIONS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    for (AppHeuristicResult appHeuristicResult : result.yarnAppHeuristicResults) {
        JsonArray detailsArray = new JsonArray();
        JsonObject heuristicResultObject = new JsonObject();
        for (AppHeuristicResultDetails details : appHeuristicResult.yarnAppHeuristicResultDetails) {
            JsonObject detailsObject = new JsonObject();
            detailsObject.addProperty(JsonKeys.NAME, details.name);
            detailsObject.addProperty(JsonKeys.VALUE, details.value);
            detailsObject.addProperty(JsonKeys.DETAILS, details.details);
            detailsArray.add(detailsObject);
        }
        heuristicResultObject.addProperty(JsonKeys.NAME, appHeuristicResult.heuristicName);
        heuristicResultObject.addProperty(JsonKeys.SEVERITY, appHeuristicResult.severity.getText());
        heuristicResultObject.add(JsonKeys.DETAILS, detailsArray);
        heuristicsArray.add(heuristicResultObject);
    }
    applicationObject.addProperty(JsonKeys.ID, result.id);
    applicationObject.addProperty(JsonKeys.USERNAME, result.username);
    applicationObject.addProperty(JsonKeys.JOB_TYPE, result.jobType);
    applicationObject.addProperty(JsonKeys.MAPREDUCE_JOB_NAME, result.jobName);
    applicationObject.addProperty(JsonKeys.START_TIME, result.startTime);
    applicationObject.addProperty(JsonKeys.FINISH_TIME, result.finishTime);
    applicationObject.addProperty(JsonKeys.RUNTIME, result.finishTime - result.startTime);
    applicationObject.addProperty(JsonKeys.WAITTIME, result.totalDelay);
    applicationObject.addProperty(JsonKeys.RESOURCE_USED, result.resourceUsed);
    applicationObject.addProperty(JsonKeys.RESOURCE_WASTED, result.resourceWasted);
    applicationObject.addProperty(JsonKeys.SEVERITY, result.severity.getText());
    applicationObject.addProperty(JsonKeys.TRACKING_URL, result.trackingUrl);
    applicationObject.addProperty(JsonKeys.JOB_EXEC_ID, result.jobExecId);
    applicationObject.addProperty(JsonKeys.JOB_DEF_ID, result.jobDefId);
    applicationObject.addProperty(JsonKeys.FLOW_EXEC_ID, result.flowExecId);
    applicationObject.addProperty(JsonKeys.FLOW_DEF_ID, result.flowDefId);
    applicationObject.addProperty(JsonKeys.QUEUE, result.queueName);
    applicationObject.add(JsonKeys.YARN_APP_HEURISTIC_RESULTS, heuristicsArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.APPLICATIONS, applicationObject);
    return ok(new Gson().toJson(parent));
}

Example 2 with AppHeuristicResultDetails

use of models.AppHeuristicResultDetails in project dr-elephant by linkedin.

the class AnalyticJob method getAnalysis.

/**
 * Returns the analysed AppResult that could be directly serialized into DB.
 *
 * This method fetches the data using the appropriate application fetcher, runs all the heuristics on them and
 * loads it into the AppResult model.
 *
 * @throws Exception if the analysis process encountered a problem.
 * @return the analysed AppResult
 */
public AppResult getAnalysis() throws Exception {
    ElephantFetcher fetcher = ElephantContext.instance().getFetcherForApplicationType(getAppType());
    HadoopApplicationData data = fetcher.fetchData(this);
    JobType jobType = ElephantContext.instance().matchJobType(data);
    String jobTypeName = jobType == null ? UNKNOWN_JOB_TYPE : jobType.getName();
    // Run all heuristics over the fetched data
    List<HeuristicResult> analysisResults = new ArrayList<HeuristicResult>();
    if (data == null || data.isEmpty()) {
        // Example: a MR job has 0 mappers and 0 reducers
        logger.info("No Data Received for analytic job: " + getAppId());
        analysisResults.add(HeuristicResult.NO_DATA);
    } else {
        List<Heuristic> heuristics = ElephantContext.instance().getHeuristicsForApplicationType(getAppType());
        for (Heuristic heuristic : heuristics) {
            String confExcludedApps = heuristic.getHeuristicConfData().getParamMap().get(EXCLUDE_JOBTYPE);
            if (confExcludedApps == null || confExcludedApps.length() == 0 || !Arrays.asList(confExcludedApps.split(",")).contains(jobTypeName)) {
                HeuristicResult result = heuristic.apply(data);
                if (result != null) {
                    analysisResults.add(result);
                }
            }
        }
    }
    HadoopMetricsAggregator hadoopMetricsAggregator = ElephantContext.instance().getAggregatorForApplicationType(getAppType());
    hadoopMetricsAggregator.aggregate(data);
    HadoopAggregatedData hadoopAggregatedData = hadoopMetricsAggregator.getResult();
    // Load app information
    AppResult result = new AppResult();
    result.id = Utils.truncateField(getAppId(), AppResult.ID_LIMIT, getAppId());
    result.trackingUrl = Utils.truncateField(getTrackingUrl(), AppResult.TRACKING_URL_LIMIT, getAppId());
    result.queueName = Utils.truncateField(getQueueName(), AppResult.QUEUE_NAME_LIMIT, getAppId());
    result.username = Utils.truncateField(getUser(), AppResult.USERNAME_LIMIT, getAppId());
    result.startTime = getStartTime();
    result.finishTime = getFinishTime();
    result.name = Utils.truncateField(getName(), AppResult.APP_NAME_LIMIT, getAppId());
    result.jobType = Utils.truncateField(jobTypeName, AppResult.JOBTYPE_LIMIT, getAppId());
    result.resourceUsed = hadoopAggregatedData.getResourceUsed();
    result.totalDelay = hadoopAggregatedData.getTotalDelay();
    result.resourceWasted = hadoopAggregatedData.getResourceWasted();
    // Load App Heuristic information
    int jobScore = 0;
    result.yarnAppHeuristicResults = new ArrayList<AppHeuristicResult>();
    Severity worstSeverity = Severity.NONE;
    for (HeuristicResult heuristicResult : analysisResults) {
        AppHeuristicResult detail = new AppHeuristicResult();
        detail.heuristicClass = Utils.truncateField(heuristicResult.getHeuristicClassName(), AppHeuristicResult.HEURISTIC_CLASS_LIMIT, getAppId());
        detail.heuristicName = Utils.truncateField(heuristicResult.getHeuristicName(), AppHeuristicResult.HEURISTIC_NAME_LIMIT, getAppId());
        detail.severity = heuristicResult.getSeverity();
        detail.score = heuristicResult.getScore();
        // Load Heuristic Details
        for (HeuristicResultDetails heuristicResultDetails : heuristicResult.getHeuristicResultDetails()) {
            AppHeuristicResultDetails heuristicDetail = new AppHeuristicResultDetails();
            heuristicDetail.yarnAppHeuristicResult = detail;
            heuristicDetail.name = Utils.truncateField(heuristicResultDetails.getName(), AppHeuristicResultDetails.NAME_LIMIT, getAppId());
            heuristicDetail.value = Utils.truncateField(heuristicResultDetails.getValue(), AppHeuristicResultDetails.VALUE_LIMIT, getAppId());
            heuristicDetail.details = Utils.truncateField(heuristicResultDetails.getDetails(), AppHeuristicResultDetails.DETAILS_LIMIT, getAppId());
            // This was added for AnalyticTest. Commenting this out to fix a bug. Also disabling AnalyticJobTest.
            // detail.yarnAppHeuristicResultDetails = new ArrayList<AppHeuristicResultDetails>();
            detail.yarnAppHeuristicResultDetails.add(heuristicDetail);
        }
        result.yarnAppHeuristicResults.add(detail);
        worstSeverity = Severity.max(worstSeverity, detail.severity);
        jobScore += detail.score;
    }
    result.severity = worstSeverity;
    result.score = jobScore;
    // Retrieve information from job configuration like scheduler information and store them into result.
    InfoExtractor.loadInfo(result, data);
    return result;
}