Examples with AppResult models.AppResult used on opensource projects

Example 26 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Web method search.

/**
 * Returns the search results for the given query
 * @return
 * JsonObject:
 *
 * <pre>
 *   {
 *         search-results: {
 *         id: "id"
 *         start: 0,
 *         end: 20,
 *         total: 0,
 *         summaries: [
 *                  {
 *                    application_summary_object
 *                  }
 *                ]
 *          }
 *  }
 * </pre>
 */
public static Result search() {
    DynamicForm form = Form.form().bindFromRequest(request());
    JsonObject parent = new JsonObject();
    int offset = SEARCH_DEFAULT_PAGE_OFFSET;
    int limit = SEARCH_DEFAULT_PAGE_LIMIT;
    int end = 0;
    int total = 0;
    if (form.get("offset") != null && form.get("offset") != "") {
        offset = Integer.valueOf(form.get("offset"));
    }
    if (form.get("limit") != null && form.get("limit") != "") {
        limit = Integer.valueOf(form.get("limit"));
    }
    if (offset < 0) {
        offset = 0;
    }
    if (limit > SEARCH_APPLICATION_MAX_OFFSET) {
        limit = SEARCH_APPLICATION_MAX_OFFSET;
    } else if (limit <= 0) {
        return ok(new Gson().toJson(parent));
    }
    Query<AppResult> query = Application.generateSearchQuery(AppResult.getSearchFields(), Application.getSearchParams());
    total = query.findRowCount();
    if (offset > total) {
        offset = total;
    }
    List<AppResult> results = query.setFirstRow(offset).setMaxRows(limit).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).findList();
    end = offset + results.size();
    JsonArray applicationSummaryArray = new JsonArray();
    for (AppResult application : results) {
        JsonObject applicationObject = new JsonObject();
        JsonArray heuristicsArray = new JsonArray();
        List<AppHeuristicResult> appHeuristicResult = application.yarnAppHeuristicResults;
        for (AppHeuristicResult heuristic : appHeuristicResult) {
            JsonObject heuristicObject = new JsonObject();
            heuristicObject.addProperty(JsonKeys.NAME, heuristic.heuristicName);
            heuristicObject.addProperty(JsonKeys.SEVERITY, heuristic.severity.getText());
            heuristicsArray.add(heuristicObject);
        }
        applicationObject.addProperty(JsonKeys.ID, application.id);
        applicationObject.addProperty(JsonKeys.USERNAME, application.username);
        applicationObject.addProperty(JsonKeys.START_TIME, application.startTime);
        applicationObject.addProperty(JsonKeys.FINISH_TIME, application.finishTime);
        applicationObject.addProperty(JsonKeys.RUNTIME, application.finishTime - application.startTime);
        applicationObject.addProperty(JsonKeys.WAITTIME, application.totalDelay);
        applicationObject.addProperty(JsonKeys.RESOURCE_USED, application.resourceUsed);
        applicationObject.addProperty(JsonKeys.RESOURCE_WASTED, application.resourceWasted);
        applicationObject.addProperty(JsonKeys.SEVERITY, application.severity.getText());
        applicationObject.addProperty(JsonKeys.QUEUE, application.queueName);
        applicationObject.add(JsonKeys.HEURISTICS_SUMMARY, heuristicsArray);
        applicationSummaryArray.add(applicationObject);
    }
    JsonObject searchResults = new JsonObject();
    searchResults.addProperty(JsonKeys.ID, query.toString());
    searchResults.addProperty(JsonKeys.START, offset);
    searchResults.addProperty(JsonKeys.END, end);
    searchResults.addProperty(JsonKeys.TOTAL, total);
    searchResults.add(JsonKeys.SUMMARIES, applicationSummaryArray);
    parent.add(JsonKeys.SEARCH_RESULTS, searchResults);
    return ok(new Gson().toJson(parent));
}

Example 27 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Web method restJobSummariesForUser.

/**
 * This method returns the json object for job-summaries for the given user
 * @param username The given username for which job-summaries json object should be returned
 * @return The job-summaries json object for the given username
 * response object:
 * <pre>
 *{
 *  "job-summaries": [
 *  {
 *    "id": "job-exec-id",
 *      "jobname": "jobname",
 *      "jobtype": "Pig",
 *      "username": "username",
 *      "starttime": 1471910835628,
 *      "finishtime": 1471911099238,
 *      "runtime": 263610,
 *      "waittime": 46234,
 *      "resourceused": 101382144,
 *      "resourcewasted": 15993417,
 *      "severity": "Moderate",
 *      "scheduler": "azkaban",
 *      "tasksseverity": [
 *    {
 *      "severity": "Moderate",
 *        "count": 1
 *    }
 *    ]
 *  }
 *  ]
 *}
 * </pre>
 */
public static Result restJobSummariesForUser(String username) {
    JsonArray jobSummaryArray = new JsonArray();
    List<AppResult> results = null;
    if (username == null || username.isEmpty()) {
        results = getSchedulerApplications(MAX_APPLICATIONS_IN_WORKFLOW);
    } else {
        results = getSchedulerApplications(username, MAX_APPLICATIONS_IN_WORKFLOW);
    }
    Map<IdUrlPair, List<AppResult>> jobExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.JOB_EXECUTION_ID), results.size(), MAX_JOB_LIMIT);
    for (IdUrlPair jobDefPair : jobExecIdToJobsMap.keySet()) {
        long totalJobMemoryUsed = 0L;
        long totalJobMemoryWasted = 0L;
        long totalJobDelay = 0L;
        long totalJobRuntime = 0L;
        long jobStartTime = Long.MAX_VALUE;
        long jobEndTime = 0;
        Severity jobSeverity = Severity.NONE;
        String jobType = null;
        String jobId = jobDefPair.getId();
        String jobName = "";
        String user = null;
        String queueName = "";
        String scheduler = "";
        String jobDefId = "";
        String jobExecId = "";
        Map<Severity, Long> applicationSeverityCount = new HashMap<Severity, Long>();
        for (AppResult application : jobExecIdToJobsMap.get(jobDefPair)) {
            totalJobMemoryUsed += application.resourceUsed;
            totalJobMemoryWasted += application.resourceWasted;
            jobType = application.jobType;
            jobName = application.jobName;
            jobDefId = application.jobDefId;
            jobExecId = application.jobExecId;
            queueName = application.queueName;
            scheduler = application.scheduler;
            if (application.startTime < jobStartTime) {
                jobStartTime = application.startTime;
            }
            if (application.finishTime > jobEndTime) {
                jobEndTime = application.finishTime;
            }
            if (application.severity.getValue() > jobSeverity.getValue()) {
                jobSeverity = application.severity;
            }
            if (applicationSeverityCount.containsKey(application.severity)) {
                applicationSeverityCount.put(application.severity, applicationSeverityCount.get(application.severity) + 1L);
            } else {
                applicationSeverityCount.put(application.severity, 1L);
            }
            user = application.username;
        }
        JsonArray applicationSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(applicationSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, applicationSeverityCount.get(key));
            applicationSeverity.add(severityObject);
        }
        totalJobDelay = Utils.getTotalWaittime(jobExecIdToJobsMap.get(jobDefPair));
        totalJobRuntime = Utils.getTotalRuntime(jobExecIdToJobsMap.get(jobDefPair));
        JsonObject jobObject = new JsonObject();
        jobObject.addProperty(JsonKeys.ID, jobId);
        jobObject.addProperty(JsonKeys.JOB_NAME, jobName);
        jobObject.addProperty(JsonKeys.JOB_TYPE, jobType);
        jobObject.addProperty(JsonKeys.USERNAME, user);
        jobObject.addProperty(JsonKeys.START_TIME, jobStartTime);
        jobObject.addProperty(JsonKeys.FINISH_TIME, jobEndTime);
        jobObject.addProperty(JsonKeys.RUNTIME, totalJobRuntime);
        jobObject.addProperty(JsonKeys.WAITTIME, totalJobDelay);
        jobObject.addProperty(JsonKeys.RESOURCE_USED, totalJobMemoryUsed);
        jobObject.addProperty(JsonKeys.RESOURCE_WASTED, totalJobMemoryWasted);
        jobObject.addProperty(JsonKeys.QUEUE, queueName);
        jobObject.addProperty(JsonKeys.SCHEDULER, scheduler);
        jobObject.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
        jobObject.addProperty(JsonKeys.JOB_DEF_ID, jobDefId);
        jobObject.addProperty(JsonKeys.JOB_EXEC_ID, jobExecId);
        jobObject.add(JsonKeys.TASKS_SEVERITY, applicationSeverity);
        jobSummaryArray.add(jobObject);
    }
    JsonArray sortedJobSummaryArray = getSortedJsonArrayByFinishTime(jobSummaryArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.JOB_SUMMARIES, sortedJobSummaryArray);
    return ok(new Gson().toJson(parent));
}

Example 28 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class AnalyticJobTest method testGetAnalysis.

@Test
public void testGetAnalysis() throws Exception {
    try {
        // Setup analytic job
        final AnalyticJob analyticJob = new AnalyticJob().setAppId(TEST_JOB_ID1).setAppType(new ApplicationType(TEST_APP_TYPE)).setFinishTime(1462178403).setStartTime(1462178412).setName(TEST_JOB_NAME).setQueueName(TEST_DEFAULT_QUEUE_NAME).setUser(TEST_USERNAME).setTrackingUrl(TEST_TRACKING_URL);
        // Setup job counter data
        String filePath = FILENAME_JOBCOUNTER;
        MapReduceCounterData jobCounter = new MapReduceCounterData();
        setCounterData(jobCounter, filePath);
        // Setup mapper data
        long[][] mapperTasksTime = { { 2563, 0, 0, 0, 0 }, { 2562, 0, 0, 0, 0 }, { 2567, 0, 0, 0, 0 } };
        MapReduceTaskData[] mappers = new MapReduceTaskData[3];
        for (int i = 1; i <= mappers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_MAPPERTASK.replaceFirst("\\$", Integer.toString(i)));
            mappers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            mappers[i - 1].setTimeAndCounter(mapperTasksTime[i - 1], taskCounter);
        }
        // Setup reducer data
        long[][] reducerTasksTime = { { 1870, 1665, 14, 0, 0 } };
        MapReduceTaskData[] reducers = new MapReduceTaskData[1];
        for (int i = 1; i <= reducers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_REDUCERTASK.replaceFirst("\\$", Integer.toString(i)));
            reducers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            reducers[i - 1].setTimeAndCounter(reducerTasksTime[i - 1], taskCounter);
        }
        // Setup job configuration data
        filePath = FILENAME_JOBCONF;
        Properties jobConf = TestUtil.loadProperties(filePath);
        // Setup application data
        final MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers).setReducerData(reducers).setJobConf(jobConf).setSucceeded(true).setDiagnosticInfo("").setUsername(TEST_USERNAME).setUrl("").setJobName(TEST_JOB_NAME).setStartTime(1462178412).setFinishTime(1462178403).setRetry(false).setAppId(TEST_JOB_ID1);
        // Setup heuristics
        final List<Heuristic> heuristics = loadHeuristics();
        // Setup job type
        final JobType jobType = new JobType(TEST_JOB_TYPE, TEST_JOBCONF_NAME, TEST_JOBCONF_PATTERN);
        // Set expectations in JMockit
        new Expectations() {

            {
                fetcher.fetchData(analyticJob);
                result = data;
                elephantContext.getHeuristicsForApplicationType(analyticJob.getAppType());
                result = heuristics;
                elephantContext.matchJobType(data);
                result = jobType;
            }
        };
        // Call the method under test
        AppResult result = analyticJob.getAnalysis();
        // Make assertions on result
        assertTrue("Result is null", result != null);
        assertTrue("Score did not match", result.score == TEST_SCORE);
        assertTrue("Severity did not match", result.severity.toString().equals(TEST_SEVERITY));
        assertTrue("APP ID did not match", result.id.equals(TEST_JOB_ID1));
        assertTrue("Scheduler did not match", result.scheduler.equals(TEST_SCHEDULER));
    } catch (Exception e) {
        e.printStackTrace();
        assertFalse("Test failed with exception", true);
    }
}

Example 29 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class AnalyticJob method getAnalysis.

/**
 * Returns the analysed AppResult that could be directly serialized into DB.
 *
 * This method fetches the data using the appropriate application fetcher, runs all the heuristics on them and
 * loads it into the AppResult model.
 *
 * @throws Exception if the analysis process encountered a problem.
 * @return the analysed AppResult
 */
public AppResult getAnalysis() throws Exception {
    ElephantFetcher fetcher = ElephantContext.instance().getFetcherForApplicationType(getAppType());
    HadoopApplicationData data = fetcher.fetchData(this);
    JobType jobType = ElephantContext.instance().matchJobType(data);
    String jobTypeName = jobType == null ? UNKNOWN_JOB_TYPE : jobType.getName();
    // Run all heuristics over the fetched data
    List<HeuristicResult> analysisResults = new ArrayList<HeuristicResult>();
    if (data == null || data.isEmpty()) {
        // Example: a MR job has 0 mappers and 0 reducers
        logger.info("No Data Received for analytic job: " + getAppId());
        analysisResults.add(HeuristicResult.NO_DATA);
    } else {
        List<Heuristic> heuristics = ElephantContext.instance().getHeuristicsForApplicationType(getAppType());
        for (Heuristic heuristic : heuristics) {
            String confExcludedApps = heuristic.getHeuristicConfData().getParamMap().get(EXCLUDE_JOBTYPE);
            if (confExcludedApps == null || confExcludedApps.length() == 0 || !Arrays.asList(confExcludedApps.split(",")).contains(jobTypeName)) {
                HeuristicResult result = heuristic.apply(data);
                if (result != null) {
                    analysisResults.add(result);
                }
            }
        }
    }
    HadoopMetricsAggregator hadoopMetricsAggregator = ElephantContext.instance().getAggregatorForApplicationType(getAppType());
    hadoopMetricsAggregator.aggregate(data);
    HadoopAggregatedData hadoopAggregatedData = hadoopMetricsAggregator.getResult();
    // Load app information
    AppResult result = new AppResult();
    result.id = Utils.truncateField(getAppId(), AppResult.ID_LIMIT, getAppId());
    result.trackingUrl = Utils.truncateField(getTrackingUrl(), AppResult.TRACKING_URL_LIMIT, getAppId());
    result.queueName = Utils.truncateField(getQueueName(), AppResult.QUEUE_NAME_LIMIT, getAppId());
    result.username = Utils.truncateField(getUser(), AppResult.USERNAME_LIMIT, getAppId());
    result.startTime = getStartTime();
    result.finishTime = getFinishTime();
    result.name = Utils.truncateField(getName(), AppResult.APP_NAME_LIMIT, getAppId());
    result.jobType = Utils.truncateField(jobTypeName, AppResult.JOBTYPE_LIMIT, getAppId());
    result.resourceUsed = hadoopAggregatedData.getResourceUsed();
    result.totalDelay = hadoopAggregatedData.getTotalDelay();
    result.resourceWasted = hadoopAggregatedData.getResourceWasted();
    // Load App Heuristic information
    int jobScore = 0;
    result.yarnAppHeuristicResults = new ArrayList<AppHeuristicResult>();
    Severity worstSeverity = Severity.NONE;
    for (HeuristicResult heuristicResult : analysisResults) {
        AppHeuristicResult detail = new AppHeuristicResult();
        detail.heuristicClass = Utils.truncateField(heuristicResult.getHeuristicClassName(), AppHeuristicResult.HEURISTIC_CLASS_LIMIT, getAppId());
        detail.heuristicName = Utils.truncateField(heuristicResult.getHeuristicName(), AppHeuristicResult.HEURISTIC_NAME_LIMIT, getAppId());
        detail.severity = heuristicResult.getSeverity();
        detail.score = heuristicResult.getScore();
        // Load Heuristic Details
        for (HeuristicResultDetails heuristicResultDetails : heuristicResult.getHeuristicResultDetails()) {
            AppHeuristicResultDetails heuristicDetail = new AppHeuristicResultDetails();
            heuristicDetail.yarnAppHeuristicResult = detail;
            heuristicDetail.name = Utils.truncateField(heuristicResultDetails.getName(), AppHeuristicResultDetails.NAME_LIMIT, getAppId());
            heuristicDetail.value = Utils.truncateField(heuristicResultDetails.getValue(), AppHeuristicResultDetails.VALUE_LIMIT, getAppId());
            heuristicDetail.details = Utils.truncateField(heuristicResultDetails.getDetails(), AppHeuristicResultDetails.DETAILS_LIMIT, getAppId());
            // This was added for AnalyticTest. Commenting this out to fix a bug. Also disabling AnalyticJobTest.
            // detail.yarnAppHeuristicResultDetails = new ArrayList<AppHeuristicResultDetails>();
            detail.yarnAppHeuristicResultDetails.add(heuristicDetail);
        }
        result.yarnAppHeuristicResults.add(detail);
        worstSeverity = Severity.max(worstSeverity, detail.severity);
        jobScore += detail.score;
    }
    result.severity = worstSeverity;
    result.score = jobScore;
    // Retrieve information from job configuration like scheduler information and store them into result.
    InfoExtractor.loadInfo(result, data);
    return result;
}