Examples with AppResult models.AppResult used on opensource projects

Example 21 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method compare.

/**
 *   Controls the Compare Feature
 */
public static Result compare() {
    DynamicForm form = Form.form().bindFromRequest(request());
    String partialFlowExecId1 = form.get(COMPARE_FLOW_ID1);
    partialFlowExecId1 = (partialFlowExecId1 != null) ? partialFlowExecId1.trim() : null;
    String partialFlowExecId2 = form.get(COMPARE_FLOW_ID2);
    partialFlowExecId2 = (partialFlowExecId2 != null) ? partialFlowExecId2.trim() : null;
    List<AppResult> results1 = null;
    List<AppResult> results2 = null;
    if (partialFlowExecId1 != null && !partialFlowExecId1.isEmpty() && partialFlowExecId2 != null && !partialFlowExecId2.isEmpty()) {
        IdUrlPair flowExecIdPair1 = bestSchedulerInfoMatchGivenPartialId(partialFlowExecId1, AppResult.TABLE.FLOW_EXEC_ID);
        IdUrlPair flowExecIdPair2 = bestSchedulerInfoMatchGivenPartialId(partialFlowExecId2, AppResult.TABLE.FLOW_EXEC_ID);
        results1 = AppResult.find.select(AppResult.getSearchFields() + "," + AppResult.TABLE.JOB_DEF_ID + "," + AppResult.TABLE.JOB_DEF_URL + "," + AppResult.TABLE.FLOW_EXEC_ID + "," + AppResult.TABLE.FLOW_EXEC_URL).where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecIdPair1.getId()).setMaxRows(100).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).findList();
        results2 = AppResult.find.select(AppResult.getSearchFields() + "," + AppResult.TABLE.JOB_DEF_ID + "," + AppResult.TABLE.JOB_DEF_URL + "," + AppResult.TABLE.FLOW_EXEC_ID + "," + AppResult.TABLE.FLOW_EXEC_URL).where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecIdPair2.getId()).setMaxRows(100).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).findList();
    }
    return ok(comparePage.render(compareResults.render(compareFlows(results1, results2))));
}

Example 22 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method restFlowGraphData.

/**
 * The data for plotting the flow history graph
 *
 * <pre>
 * {@code
 *   [
 *     {
 *       "flowtime": <Last job's finish time>,
 *       "score": 1000,
 *       "jobscores": [
 *         {
 *           "jobdefurl:" "url",
 *           "jobexecurl:" "url",
 *           "jobscore": 500
 *         },
 *         {
 *           "jobdefurl:" "url",
 *           "jobexecurl:" "url",
 *           "jobscore": 500
 *         }
 *       ]
 *     },
 *     {
 *       "flowtime": <Last job's finish time>,
 *       "score": 700,
 *       "jobscores": [
 *         {
 *           "jobdefurl:" "url",
 *           "jobexecurl:" "url",
 *           "jobscore": 0
 *         },
 *         {
 *           "jobdefurl:" "url",
 *           "jobexecurl:" "url",
 *           "jobscore": 700
 *         }
 *       ]
 *     }
 *   ]
 * }
 * </pre>
 */
public static Result restFlowGraphData(String flowDefId) {
    JsonArray datasets = new JsonArray();
    if (flowDefId == null || flowDefId.isEmpty()) {
        return ok(new Gson().toJson(datasets));
    }
    // Fetch available flow executions with latest JOB_HISTORY_LIMIT mr jobs.
    List<AppResult> results = getRestFlowAppResults(flowDefId);
    if (results.size() == 0) {
        logger.info("No results for Job url");
    }
    Map<IdUrlPair, List<AppResult>> flowExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID), results.size(), MAX_HISTORY_LIMIT);
    // Compute the graph data starting from the earliest available execution to latest
    List<IdUrlPair> keyList = new ArrayList<IdUrlPair>(flowExecIdToJobsMap.keySet());
    for (int i = keyList.size() - 1; i >= 0; i--) {
        IdUrlPair flowExecPair = keyList.get(i);
        int flowPerfScore = 0;
        JsonArray jobScores = new JsonArray();
        List<AppResult> mrJobsList = Lists.reverse(flowExecIdToJobsMap.get(flowExecPair));
        Map<IdUrlPair, List<AppResult>> jobDefIdToJobsMap = ControllerUtil.groupJobs(mrJobsList, ControllerUtil.GroupBy.JOB_DEFINITION_ID);
        // Compute the execution records. Note that each entry in the jobDefIdToJobsMap will have at least one AppResult
        for (IdUrlPair jobDefPair : jobDefIdToJobsMap.keySet()) {
            // Compute job perf score
            int jobPerfScore = 0;
            for (AppResult job : jobDefIdToJobsMap.get(jobDefPair)) {
                jobPerfScore += job.score;
            }
            // A job in jobscores list
            JsonObject jobScore = new JsonObject();
            jobScore.addProperty("jobscore", jobPerfScore);
            jobScore.addProperty("jobdefurl", jobDefPair.getUrl());
            jobScore.addProperty("jobexecurl", jobDefIdToJobsMap.get(jobDefPair).get(0).jobExecUrl);
            jobScores.add(jobScore);
            flowPerfScore += jobPerfScore;
        }
        // Execution record
        JsonObject dataset = new JsonObject();
        dataset.addProperty("flowtime", Utils.getFlowTime(mrJobsList.get(mrJobsList.size() - 1)));
        dataset.addProperty("score", flowPerfScore);
        dataset.add("jobscores", jobScores);
        datasets.add(dataset);
    }
    JsonArray sortedDatasets = Utils.sortJsonArray(datasets);
    return ok(new Gson().toJson(sortedDatasets));
}

Example 23 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method getUserResourceUsage.

/**
 * Returns the list of users with their resourceUsed and resourceWasted Data for the given time range
 * @return list of AppResourceUsageData
 */
private static Collection<AppResourceUsageData> getUserResourceUsage(Date start, Date end) {
    long resourceUsed = 0;
    Map<String, AppResourceUsageData> userResourceUsage = new HashMap<String, AppResourceUsageData>();
    // Fetch all the appresults for the given time range [startTime, endTime).
    List<AppResult> results = AppResult.find.select("*").where().ge(AppResult.TABLE.START_TIME, start.getTime()).lt(AppResult.TABLE.START_TIME, end.getTime()).findList();
    // aggregate the resourceUsage data at the user level
    for (AppResult result : results) {
        if (!userResourceUsage.containsKey(result.username)) {
            AppResourceUsageData data = new AppResourceUsageData();
            data.user = result.username;
            userResourceUsage.put(result.username, data);
        }
        userResourceUsage.get(result.username).resourceUsed += Utils.MBSecondsToGBHours(result.resourceUsed);
        userResourceUsage.get(result.username).resourceWasted += Utils.MBSecondsToGBHours(result.resourceWasted);
    }
    return userResourceUsage.values();
}

Example 24 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class ControllerUtil method groupJobs.

/**
 * Grouping a list of AppResult by GroupBy enum.
 *
 * @param results The list of jobs of type AppResult to be grouped.
 * @param groupBy The field by which the results have to be grouped.
 * @return A map with the grouped field as the key and the list of jobs as the value.
 */
public static Map<IdUrlPair, List<AppResult>> groupJobs(List<AppResult> results, GroupBy groupBy) {
    Map<String, List<AppResult>> groupMap = new LinkedHashMap<String, List<AppResult>>();
    Map<String, String> idUrlMap = new HashMap<String, String>();
    for (AppResult result : results) {
        String idField = null;
        String urlField = null;
        switch(groupBy) {
            case JOB_EXECUTION_ID:
                idField = result.jobExecId;
                urlField = result.jobExecUrl;
                break;
            case JOB_DEFINITION_ID:
                idField = result.jobDefId;
                urlField = result.jobDefUrl;
                break;
            case FLOW_EXECUTION_ID:
                idField = result.flowExecId;
                urlField = result.flowExecUrl;
                break;
        }
        if (!idUrlMap.containsKey(idField)) {
            idUrlMap.put(idField, urlField);
        }
        if (groupMap.containsKey(idField)) {
            groupMap.get(idField).add(result);
        } else {
            List<AppResult> list = new ArrayList<AppResult>();
            list.add(result);
            groupMap.put(idField, list);
        }
    }
    // Construct the final result map with the key as a (id, url) pair.
    Map<IdUrlPair, List<AppResult>> resultMap = new LinkedHashMap<IdUrlPair, List<AppResult>>();
    for (Map.Entry<String, List<AppResult>> entry : groupMap.entrySet()) {
        String key = entry.getKey();
        List<AppResult> value = entry.getValue();
        resultMap.put(new IdUrlPair(key, idUrlMap.get(key)), value);
    }
    return resultMap;
}

Example 25 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class FitnessComputeUtil method updateExecutionMetrics.

/**
 * Updates the execution metrics
 * @param completedExecutions List of completed executions
 */
protected void updateExecutionMetrics(List<TuningJobExecution> completedExecutions) {
    for (TuningJobExecution tuningJobExecution : completedExecutions) {
        logger.info("Updating execution metrics and fitness for execution: " + tuningJobExecution.jobExecution.jobExecId);
        try {
            JobExecution jobExecution = tuningJobExecution.jobExecution;
            JobDefinition job = jobExecution.job;
            // job id match and tuning enabled
            TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").fetch(TuningJobDefinition.TABLE.job, "*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id).eq(TuningJobDefinition.TABLE.tuningEnabled, 1).findUnique();
            List<AppResult> results = AppResult.find.select("*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").where().eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId).eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId).findList();
            if (results != null && results.size() > 0) {
                Long totalExecutionTime = 0L;
                Double totalResourceUsed = 0D;
                Double totalInputBytesInBytes = 0D;
                for (AppResult appResult : results) {
                    totalResourceUsed += appResult.resourceUsed;
                    totalInputBytesInBytes += getTotalInputBytes(appResult);
                }
                Long totalRunTime = Utils.getTotalRuntime(results);
                Long totalDelay = Utils.getTotalWaittime(results);
                totalExecutionTime = totalRunTime - totalDelay;
                if (totalExecutionTime != 0) {
                    jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60);
                    jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600);
                    jobExecution.inputSizeInBytes = totalInputBytesInBytes;
                    logger.info("Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes);
                }
                if (tuningJobDefinition.averageResourceUsage == null && totalExecutionTime != 0) {
                    tuningJobDefinition.averageResourceUsage = jobExecution.resourceUsage;
                    tuningJobDefinition.averageExecutionTime = jobExecution.executionTime;
                    tuningJobDefinition.averageInputSizeInBytes = jobExecution.inputSizeInBytes.longValue();
                    tuningJobDefinition.update();
                }
                // Compute fitness
                if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals(JobExecution.ExecutionState.CANCELLED)) {
                    logger.info("Execution " + jobExecution.jobExecId + " failed/cancelled. Applying penalty");
                    // Todo: Check if the reason of failure is auto tuning and  handle cancelled cases
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * tuningJobDefinition.averageInputSizeInBytes);
                } else if (jobExecution.resourceUsage > (// Todo: Check execution time constraint as well
                tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0)) {
                    logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage");
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * totalInputBytesInBytes);
                } else {
                    tuningJobExecution.fitness = jobExecution.resourceUsage * FileUtils.ONE_GB / totalInputBytesInBytes;
                }
                tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED;
                jobExecution.update();
                tuningJobExecution.update();
            } else {
                if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals(JobExecution.ExecutionState.CANCELLED)) {
                    // Todo: Check if the reason of failure is auto tuning and  handle cancelled cases
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * tuningJobDefinition.averageInputSizeInBytes);
                    jobExecution.executionTime = 0D;
                    jobExecution.resourceUsage = 0D;
                    jobExecution.inputSizeInBytes = 0D;
                    tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED;
                    jobExecution.update();
                    tuningJobExecution.update();
                }
            }
        } catch (Exception e) {
            logger.error("Error updating fitness of execution: " + tuningJobExecution.jobExecution.id + "\n Stacktrace: ", e);
        }
    }
    logger.info("Execution metrics updated");
}