Examples with AppResult models.AppResult used on opensource projects

Example 16 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method restCompare.

/**
 * The Rest API for Compare Feature
 * E.g., localhost:8080/rest/compare?flow-exec-id1=abc&flow-exec-id2=xyz
 */
public static Result restCompare() {
    DynamicForm form = Form.form().bindFromRequest(request());
    String flowExecId1 = form.get(COMPARE_FLOW_ID1);
    flowExecId1 = (flowExecId1 != null) ? flowExecId1.trim() : null;
    String flowExecId2 = form.get(COMPARE_FLOW_ID2);
    flowExecId2 = (flowExecId2 != null) ? flowExecId2.trim() : null;
    List<AppResult> results1 = null;
    List<AppResult> results2 = null;
    if (flowExecId1 != null && !flowExecId1.isEmpty() && flowExecId2 != null && !flowExecId2.isEmpty()) {
        results1 = AppResult.find.select("*").where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecId1).setMaxRows(100).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").findList();
        results2 = AppResult.find.select("*").where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecId2).setMaxRows(100).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").findList();
    }
    Map<IdUrlPair, Map<IdUrlPair, List<AppResult>>> compareResults = compareFlows(results1, results2);
    Map<String, Map<String, List<AppResult>>> resMap = new HashMap<String, Map<String, List<AppResult>>>();
    for (Map.Entry<IdUrlPair, Map<IdUrlPair, List<AppResult>>> entry : compareResults.entrySet()) {
        IdUrlPair jobExecPair = entry.getKey();
        Map<IdUrlPair, List<AppResult>> value = entry.getValue();
        for (Map.Entry<IdUrlPair, List<AppResult>> innerEntry : value.entrySet()) {
            IdUrlPair flowExecPair = innerEntry.getKey();
            List<AppResult> results = innerEntry.getValue();
            Map<String, List<AppResult>> resultMap = new HashMap<String, List<AppResult>>();
            resultMap.put(flowExecPair.getId(), results);
            resMap.put(jobExecPair.getId(), resultMap);
        }
    }
    return ok(Json.toJson(resMap));
}

Example 17 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method restSearch.

/**
 * The Rest API for Search Feature
 *
 * http://localhost:8080/rest/search?username=abc&job-type=HadoopJava
 */
public static Result restSearch() {
    DynamicForm form = Form.form().bindFromRequest(request());
    String appId = form.get(APP_ID);
    appId = appId != null ? appId.trim() : "";
    if (appId.contains("job")) {
        appId = appId.replaceAll("job", "application");
    }
    String flowExecId = form.get(FLOW_EXEC_ID);
    flowExecId = (flowExecId != null) ? flowExecId.trim() : null;
    if (!appId.isEmpty()) {
        AppResult result = AppResult.find.select("*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").where().idEq(appId).findUnique();
        if (result != null) {
            return ok(Json.toJson(result));
        } else {
            return notFound("Unable to find record on id: " + appId);
        }
    } else if (flowExecId != null && !flowExecId.isEmpty()) {
        List<AppResult> results = AppResult.find.select("*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecId).findList();
        if (results.size() == 0) {
            return notFound("Unable to find record on flow execution: " + flowExecId);
        } else {
            return ok(Json.toJson(results));
        }
    }
    int page = 1;
    if (request().queryString().containsKey(PAGE)) {
        page = Integer.parseInt(request().queryString().get(PAGE)[0]);
        if (page <= 0) {
            page = 1;
        }
    }
    Query<AppResult> query = generateSearchQuery("*", getSearchParams());
    List<AppResult> results = query.setFirstRow((page - 1) * REST_PAGE_LENGTH).setMaxRows(REST_PAGE_LENGTH).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").findList();
    if (results.size() == 0) {
        return notFound("No records");
    } else {
        return ok(Json.toJson(results));
    }
}

Example 18 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method search.

/**
 * Controls the Search Feature
 */
public static Result search() {
    DynamicForm form = Form.form().bindFromRequest(request());
    String appId = form.get(APP_ID);
    appId = appId != null ? appId.trim() : "";
    if (appId.contains("job")) {
        appId = appId.replaceAll("job", "application");
    }
    String partialFlowExecId = form.get(FLOW_EXEC_ID);
    partialFlowExecId = (partialFlowExecId != null) ? partialFlowExecId.trim() : null;
    String jobDefId = form.get(JOB_DEF_ID);
    jobDefId = jobDefId != null ? jobDefId.trim() : "";
    // Search and display job details when job id or flow execution url is provided.
    if (!appId.isEmpty()) {
        AppResult result = AppResult.find.select("*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*").fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*").where().idEq(appId).findUnique();
        return ok(searchPage.render(null, jobDetails.render(result)));
    } else if (Utils.isSet(partialFlowExecId)) {
        IdUrlPair flowExecPair = bestSchedulerInfoMatchGivenPartialId(partialFlowExecId, AppResult.TABLE.FLOW_EXEC_ID);
        List<AppResult> results = AppResult.find.select(AppResult.getSearchFields() + "," + AppResult.TABLE.JOB_EXEC_ID).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).where().eq(AppResult.TABLE.FLOW_EXEC_ID, flowExecPair.getId()).findList();
        Map<IdUrlPair, List<AppResult>> map = ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.JOB_EXECUTION_ID);
        return ok(searchPage.render(null, flowDetails.render(flowExecPair, map)));
    } else if (!jobDefId.isEmpty()) {
        List<AppResult> results = AppResult.find.select(AppResult.getSearchFields() + "," + AppResult.TABLE.JOB_DEF_ID).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).where().eq(AppResult.TABLE.JOB_DEF_ID, jobDefId).findList();
        Map<IdUrlPair, List<AppResult>> map = ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID);
        // all results should have the same flow id
        String flowDefId = (results.isEmpty()) ? "" : results.get(0).flowDefId;
        IdUrlPair flowDefIdPair = new IdUrlPair(flowDefId, AppResult.TABLE.FLOW_DEF_URL);
        return ok(searchPage.render(null, flowDefinitionIdDetails.render(flowDefIdPair, map)));
    }
    // Prepare pagination of results
    PaginationStats paginationStats = new PaginationStats(PAGE_LENGTH, PAGE_BAR_LENGTH);
    int pageLength = paginationStats.getPageLength();
    paginationStats.setCurrentPage(1);
    final Map<String, String[]> searchString = request().queryString();
    if (searchString.containsKey(PAGE)) {
        try {
            paginationStats.setCurrentPage(Integer.parseInt(searchString.get(PAGE)[0]));
        } catch (NumberFormatException ex) {
            logger.error("Error parsing page number. Setting current page to 1.");
            paginationStats.setCurrentPage(1);
        }
    }
    int currentPage = paginationStats.getCurrentPage();
    int paginationBarStartIndex = paginationStats.getPaginationBarStartIndex();
    // Filter jobs by search parameters
    Query<AppResult> query = generateSearchQuery(AppResult.getSearchFields(), getSearchParams());
    List<AppResult> results = query.setFirstRow((paginationBarStartIndex - 1) * pageLength).setMaxRows((paginationStats.getPageBarLength() - 1) * pageLength + 1).fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, AppHeuristicResult.getSearchFields()).findList();
    paginationStats.setQueryString(getQueryString());
    if (results.isEmpty() || currentPage > paginationStats.computePaginationBarEndIndex(results.size())) {
        return ok(searchPage.render(null, jobDetails.render(null)));
    } else {
        List<AppResult> resultsToDisplay = results.subList((currentPage - paginationBarStartIndex) * pageLength, Math.min(results.size(), (currentPage - paginationBarStartIndex + 1) * pageLength));
        return ok(searchPage.render(paginationStats, searchResults.render(String.format("Results: Showing %,d of %,d", resultsToDisplay.size(), query.findRowCount()), resultsToDisplay)));
    }
}

Example 19 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method restJobGraphData.

/**
 * The data for plotting the job history graph. While plotting the job history
 * graph an ajax call is made to this to fetch the graph data.
 *
 * Data Returned:
 * <pre>
 * {@code
 *   [
 *     {
 *       "flowtime": <Last job's finish time>,
 *       "score": 1000,
 *       "stagescores": [
 *         {
 *           "stageid:" "id",
 *           "stagescore": 500
 *         },
 *         {
 *           "stageid:" "id",
 *           "stagescore": 500
 *         }
 *       ]
 *     },
 *     {
 *       "flowtime": <Last job's finish time>,
 *       "score": 700,
 *       "stagescores": [
 *         {
 *           "stageid:" "id",
 *           "stagescore": 0
 *         },
 *         {
 *           "stageid:" "id",
 *           "stagescore": 700
 *         }
 *       ]
 *     }
 *   ]
 * }
 * </pre>
 */
public static Result restJobGraphData(String jobDefId) {
    JsonArray datasets = new JsonArray();
    if (jobDefId == null || jobDefId.isEmpty()) {
        return ok(new Gson().toJson(datasets));
    }
    // Fetch available flow executions with latest JOB_HISTORY_LIMIT mr jobs.
    List<AppResult> results = getRestJobAppResults(jobDefId);
    if (results.size() == 0) {
        logger.info("No results for Job url");
    }
    Map<IdUrlPair, List<AppResult>> flowExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID), results.size(), MAX_HISTORY_LIMIT);
    // Compute the graph data starting from the earliest available execution to latest
    List<IdUrlPair> keyList = new ArrayList<IdUrlPair>(flowExecIdToJobsMap.keySet());
    for (int i = keyList.size() - 1; i >= 0; i--) {
        IdUrlPair flowExecPair = keyList.get(i);
        int jobPerfScore = 0;
        JsonArray stageScores = new JsonArray();
        List<AppResult> mrJobsList = Lists.reverse(flowExecIdToJobsMap.get(flowExecPair));
        for (AppResult appResult : flowExecIdToJobsMap.get(flowExecPair)) {
            // Each MR job triggered by jobDefId for flowExecId
            int mrPerfScore = 0;
            for (AppHeuristicResult appHeuristicResult : appResult.yarnAppHeuristicResults) {
                mrPerfScore += appHeuristicResult.score;
            }
            // A particular mr stage
            JsonObject stageScore = new JsonObject();
            stageScore.addProperty("stageid", appResult.id);
            stageScore.addProperty("stagescore", mrPerfScore);
            stageScores.add(stageScore);
            jobPerfScore += mrPerfScore;
        }
        // Execution record
        JsonObject dataset = new JsonObject();
        dataset.addProperty("flowtime", Utils.getFlowTime(mrJobsList.get(mrJobsList.size() - 1)));
        dataset.addProperty("score", jobPerfScore);
        dataset.add("stagescores", stageScores);
        datasets.add(dataset);
    }
    JsonArray sortedDatasets = Utils.sortJsonArray(datasets);
    return ok(new Gson().toJson(sortedDatasets));
}

Example 20 with AppResult

use of models.AppResult in project dr-elephant by linkedin.

the class Application method restFlowMetricsGraphData.

/**
 * Rest data to plot flot history graph using time and resource metrics. While plotting the flow history
 * graph an ajax call is made to this to fetch the graph data.
 * [
 * {
 *  "flowtime": 1461744881991,
 *  "runtime": 3190223,
 *  "waittime": 368011,
 *  "resourceused": 180488192,
 *  "resourcewasted": 0,
 *  "jobmetrics": [
 *          {
 *         "runtime": 3190223,
 *         "waittime": 368011,
 *         "resourceused": 180488192,
 *         "resourcewasted": 0,
 *         "jobdefurl": "sampleURL"
 *         "jobexecurl": "sampleURL"
 *          }
 *        ]
 * },
 * {
 *  "flowtime": 1461818409959,
 *  "runtime": 897490,
 *  "waittime": 100703,
 *  "resourceused": 12863488,
 *  "resourcewasted": 0,
 *  "jobmetrics": [
 *          {
 *         "runtime": 897490,
 *         "waittime": 100703,
 *         "resourceused": 12863488,
 *         "resourcewasted": 0,
 *         "jobdefurl": "sampleURL"
 *         "jobexecurl": "sampleURL"
 * }
 * ]
 *}
 *]
 */
public static Result restFlowMetricsGraphData(String flowDefId) {
    JsonArray datasets = new JsonArray();
    if (flowDefId == null || flowDefId.isEmpty()) {
        return ok(new Gson().toJson(datasets));
    }
    List<AppResult> results = getRestFlowAppResults(flowDefId);
    if (results.size() == 0) {
        logger.info("No results for Job url");
    }
    Map<IdUrlPair, List<AppResult>> flowExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID), results.size(), MAX_HISTORY_LIMIT);
    // Compute the graph data starting from the earliest available execution to latest
    List<IdUrlPair> keyList = new ArrayList<IdUrlPair>(flowExecIdToJobsMap.keySet());
    for (int i = keyList.size() - 1; i >= 0; i--) {
        IdUrlPair flowExecPair = keyList.get(i);
        int flowPerfScore = 0;
        JsonArray jobScores = new JsonArray();
        List<AppResult> mrJobsList = Lists.reverse(flowExecIdToJobsMap.get(flowExecPair));
        Map<IdUrlPair, List<AppResult>> jobDefIdToJobsMap = ControllerUtil.groupJobs(mrJobsList, ControllerUtil.GroupBy.JOB_DEFINITION_ID);
        long totalFlowMemoryUsed = 0;
        long totalFlowMemoryWasted = 0;
        long totalFlowDelay = 0;
        long totalFlowRuntime = 0;
        // Compute the execution records. Note that each entry in the jobDefIdToJobsMap will have at least one AppResult
        for (IdUrlPair jobDefPair : jobDefIdToJobsMap.keySet()) {
            // Compute job perf score
            long totalJobMemoryUsed = 0;
            long totalJobMemoryWasted = 0;
            long totalJobDelay = 0;
            long totalJobRuntime = 0;
            totalJobRuntime = Utils.getTotalRuntime(jobDefIdToJobsMap.get(jobDefPair));
            totalJobDelay = Utils.getTotalWaittime(jobDefIdToJobsMap.get(jobDefPair));
            for (AppResult job : jobDefIdToJobsMap.get(jobDefPair)) {
                totalJobMemoryUsed += job.resourceUsed;
                totalJobMemoryWasted += job.resourceWasted;
            }
            // A job in jobscores list
            JsonObject jobScore = new JsonObject();
            jobScore.addProperty("runtime", totalJobRuntime);
            jobScore.addProperty("waittime", totalJobDelay);
            jobScore.addProperty("resourceused", totalJobMemoryUsed);
            jobScore.addProperty("resourcewasted", totalJobMemoryWasted);
            jobScore.addProperty("jobdefurl", jobDefPair.getUrl());
            jobScore.addProperty("jobexecurl", jobDefIdToJobsMap.get(jobDefPair).get(0).jobExecUrl);
            jobScores.add(jobScore);
            totalFlowMemoryUsed += totalJobMemoryUsed;
            totalFlowMemoryWasted += totalJobMemoryWasted;
        }
        totalFlowDelay = Utils.getTotalWaittime(flowExecIdToJobsMap.get(flowExecPair));
        totalFlowRuntime = Utils.getTotalRuntime(flowExecIdToJobsMap.get(flowExecPair));
        // Execution record
        JsonObject dataset = new JsonObject();
        dataset.addProperty("flowtime", Utils.getFlowTime(mrJobsList.get(mrJobsList.size() - 1)));
        dataset.addProperty("runtime", totalFlowRuntime);
        dataset.addProperty("waittime", totalFlowDelay);
        dataset.addProperty("resourceused", totalFlowMemoryUsed);
        dataset.addProperty("resourcewasted", totalFlowMemoryWasted);
        dataset.add("jobmetrics", jobScores);
        datasets.add(dataset);
    }
    JsonArray sortedDatasets = Utils.sortJsonArray(datasets);
    return ok(new Gson().toJson(sortedDatasets));
}