Examples with IdUrlPair controllers.IdUrlPair used on opensource projects

Example 1 with IdUrlPair

use of controllers.IdUrlPair in project dr-elephant by linkedin.

the class Web method restWorkflowFromFlowId.

/**
 * This method returns the workflow response object based on the flow execution id
 * @param flowId The flow execution id for which the flow should be returned
 * @return Return the workflow detail based on the flow execution id
 *
 * response object:
 * <pre>
 * *{
 *    "workflows": {
 *    "id": "flowid",
 *        "username": "username",
 *        "starttime": 1471910835628,
 *        "finishtime": 1471911099238,
 *        "runtime": 263610,
 *        "waittime": 46234,
 *        "resourceused": 101382144,
 *        "resourcewasted": 15993417,
 *        "severity": "Moderate",
 *        "flowexecid": "flowexecid",
 *        "flowdefid": "flowdefid",
 *        "jobssummaries": [
 *          {
 *            "id": "jobid",
 *            "jobname": "jobname",
 *            "jobtype": "Pig",
 *            "username": "username",
 *            "starttime": 1471910835628,
 *            "finishtime": 1471911099238,
 *            "runtime": 263610,
 *            "waittime": 46234,
 *            "resourceused": 101382144,
 *            "resourcewasted": 15993417,
 *            "severity": "Moderate",
 *            "tasksseverity": [
 *              {
 *                "severity": "Moderate",
 *                "count": 1
 *              }
 *             ]
 *        }
 *      ],
 *        "jobsseverity": [
 *          {
 *            "severity": "Moderate",
 *            "count": 1
 *          }
 *      ]
 *  }
 *}
 * </pre>
 */
public static Result restWorkflowFromFlowId(String flowId) {
    if (flowId == null || flowId.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.WORKFLOWS, new JsonObject());
        return notFound(new Gson().toJson(parent));
    }
    JsonArray jobSeverityArray = new JsonArray();
    JsonArray jobSummaryArray = new JsonArray();
    JsonObject data = new JsonObject();
    String flowExecId = flowId;
    String username = "";
    long totalFlowResourceUsed = 0;
    long totalFlowResourceWasted = 0;
    long totalFlowRuntime = 0;
    long totalFlowDelay = 0;
    Severity flowSeverity = Severity.NONE;
    long flowStartTime = Long.MAX_VALUE;
    long flowEndTime = 0;
    String flowDefinitionId = "";
    Map<Severity, Long> jobSeverityCount = new HashMap<Severity, Long>();
    String wfQueueName = "";
    String wfSchedulerName = "";
    List<AppResult> results = getRestFlowResultsFromFlowExecutionId(flowId);
    if (results.isEmpty()) {
        JsonObject parent = new JsonObject();
        parent.add(JsonKeys.WORKFLOWS, data);
        return notFound(new Gson().toJson(parent));
    }
    Map<IdUrlPair, List<AppResult>> jobExecIdToJobsMap = ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.JOB_EXECUTION_ID);
    for (IdUrlPair jobDefPair : jobExecIdToJobsMap.keySet()) {
        long totalJobMemoryUsed = 0;
        long totalJobMemoryWasted = 0;
        long totalJobDelay = 0;
        long totalJobRuntime = 0;
        long jobStartTime = Long.MAX_VALUE;
        long jobEndTime = 0;
        Severity jobSeverity = Severity.NONE;
        String jobType = null;
        String jobId = jobDefPair.getId();
        String jobName = "";
        String queueName = "";
        String schedulerName = "";
        Map<Severity, Long> taskSeverityCount = new HashMap<Severity, Long>();
        for (AppResult task : jobExecIdToJobsMap.get(jobDefPair)) {
            totalJobMemoryUsed += task.resourceUsed;
            totalJobMemoryWasted += task.resourceWasted;
            username = task.username;
            jobType = task.jobType;
            jobName = task.jobName;
            flowDefinitionId = task.flowDefId;
            queueName = task.queueName;
            schedulerName = task.scheduler;
            if (task.startTime < jobStartTime) {
                jobStartTime = task.startTime;
            }
            if (task.finishTime > jobEndTime) {
                jobEndTime = task.finishTime;
            }
            if (task.severity.getValue() > jobSeverity.getValue()) {
                jobSeverity = task.severity;
            }
            if (taskSeverityCount.containsKey(task.severity)) {
                taskSeverityCount.put(task.severity, taskSeverityCount.get(task.severity) + 1L);
            } else {
                taskSeverityCount.put(task.severity, 1L);
            }
        }
        if (jobSeverityCount.containsKey(jobSeverity)) {
            jobSeverityCount.put(jobSeverity, jobSeverityCount.get(jobSeverity) + 1L);
        } else {
            jobSeverityCount.put(jobSeverity, 1L);
        }
        JsonArray taskSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(taskSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, taskSeverityCount.get(key));
            taskSeverity.add(severityObject);
        }
        wfQueueName = queueName;
        wfSchedulerName = schedulerName;
        totalJobDelay = Utils.getTotalWaittime(jobExecIdToJobsMap.get(jobDefPair));
        totalJobRuntime = Utils.getTotalRuntime(jobExecIdToJobsMap.get(jobDefPair));
        JsonObject jobObject = new JsonObject();
        jobObject.addProperty(JsonKeys.ID, jobId);
        jobObject.addProperty(JsonKeys.JOB_NAME, jobName);
        jobObject.addProperty(JsonKeys.JOB_TYPE, jobType);
        jobObject.addProperty(JsonKeys.USERNAME, username);
        jobObject.addProperty(JsonKeys.START_TIME, jobStartTime);
        jobObject.addProperty(JsonKeys.FINISH_TIME, jobEndTime);
        jobObject.addProperty(JsonKeys.RUNTIME, totalJobRuntime);
        jobObject.addProperty(JsonKeys.WAITTIME, totalJobDelay);
        jobObject.addProperty(JsonKeys.RESOURCE_USED, totalJobMemoryUsed);
        jobObject.addProperty(JsonKeys.RESOURCE_WASTED, totalJobMemoryWasted);
        jobObject.addProperty(JsonKeys.QUEUE, queueName);
        jobObject.addProperty(JsonKeys.SCHEDULER, schedulerName);
        jobObject.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
        jobObject.add(JsonKeys.TASKS_SEVERITY, taskSeverity);
        jobSummaryArray.add(jobObject);
        totalFlowResourceUsed += totalJobMemoryUsed;
        totalFlowResourceWasted += totalJobMemoryWasted;
        if (jobSeverity.getValue() > flowSeverity.getValue()) {
            flowSeverity = jobSeverity;
        }
        if (flowStartTime > jobStartTime) {
            flowStartTime = jobStartTime;
        }
        if (flowEndTime < jobEndTime) {
            flowEndTime = jobEndTime;
        }
    }
    // job map scope ends here
    List<Severity> keys = getSortedSeverityKeys(jobSeverityCount.keySet());
    for (Severity key : keys) {
        JsonObject severityObject = new JsonObject();
        severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
        severityObject.addProperty(JsonKeys.COUNT, jobSeverityCount.get(key));
        jobSeverityArray.add(severityObject);
    }
    totalFlowDelay = Utils.getTotalWaittime(results);
    totalFlowRuntime = Utils.getTotalRuntime(results);
    data.addProperty(JsonKeys.ID, flowExecId);
    data.addProperty(JsonKeys.USERNAME, username);
    data.addProperty(JsonKeys.START_TIME, flowStartTime);
    data.addProperty(JsonKeys.FINISH_TIME, flowEndTime);
    data.addProperty(JsonKeys.RUNTIME, totalFlowRuntime);
    data.addProperty(JsonKeys.WAITTIME, totalFlowDelay);
    data.addProperty(JsonKeys.RESOURCE_USED, totalFlowResourceUsed);
    data.addProperty(JsonKeys.RESOURCE_WASTED, totalFlowResourceWasted);
    data.addProperty(JsonKeys.SEVERITY, flowSeverity.getText());
    data.addProperty(JsonKeys.FLOW_EXEC_ID, flowExecId);
    data.addProperty(JsonKeys.FLOW_DEF_ID, flowDefinitionId);
    data.addProperty(JsonKeys.QUEUE, wfQueueName);
    data.addProperty(JsonKeys.SCHEDULER, wfSchedulerName);
    data.add(JsonKeys.JOBSSUMMARIES, jobSummaryArray);
    data.add(JsonKeys.JOBS_SEVERITY, jobSeverityArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.WORKFLOWS, data);
    return ok(new Gson().toJson(parent));
}

Example 2 with IdUrlPair

use of controllers.IdUrlPair in project dr-elephant by linkedin.

the class Web method restWorkflowSummariesForUser.

/**
 * This method returns the workflow-summaries json response
 * @param username The username for which workflow-summaries must be returned
 * @return The json response of the workflow-summaries for the given user
 * Response data:
 * <pre>
 *{
 *  "workflow-summaries": [
 *  {
 *    "id": "http://workflow-id",
 *      "username": "search",
 *      "starttime": 1468818098875,
 *      "finishtime": 1468819946683,
 *      "runtime": 1855532,
 *      "waittime": 365368,
 *      "resourceused": 3306438656,
 *      "resourcewasted": 516978829,
 *      "severity": "Severe",
 *      "jobsseverity": [
 *    {
 *      "severity": "Severe",
 *        "count": 26
 *    },
 *    {
 *      "severity": "Moderate",
 *        "count": 3
 *    },
 *    {
 *      "severity": "Low",
 *        "count": 1
 *    },
 *    {
 *      "severity": "None",
 *        "count": 16
 *    }
 *    ]
 *  }
 *  ]
 *}
 * </pre>
 */
public static Result restWorkflowSummariesForUser(String username) {
    JsonArray workflowSummaryArray = new JsonArray();
    List<AppResult> results = null;
    if (username == null || username.isEmpty()) {
        results = getSchedulerApplications(MAX_APPLICATIONS_IN_WORKFLOW);
    } else {
        results = getSchedulerApplications(username, MAX_APPLICATIONS_IN_WORKFLOW);
    }
    Map<IdUrlPair, List<AppResult>> flowExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.FLOW_EXECUTION_ID), results.size(), MAX_FLOW_LIMIT);
    List<IdUrlPair> keyList = new ArrayList<IdUrlPair>(flowExecIdToJobsMap.keySet());
    for (IdUrlPair flowExecPair : keyList) {
        List<AppResult> mrJobsList = Lists.reverse(flowExecIdToJobsMap.get(flowExecPair));
        Map<IdUrlPair, List<AppResult>> jobDefIdToJobsMap = ControllerUtil.groupJobs(mrJobsList, ControllerUtil.GroupBy.JOB_EXECUTION_ID);
        Map<Severity, Long> jobSeverityCount = new HashMap<Severity, Long>();
        long totalFlowMemoryUsed = 0;
        long totalFlowMemoryWasted = 0;
        long totalFlowDelay = 0;
        long totalFlowRuntime = 0;
        Severity flowSeverity = Severity.NONE;
        for (IdUrlPair jobDefPair : jobDefIdToJobsMap.keySet()) {
            Severity jobseverity = Severity.NONE;
            long totalJobMemoryUsed = 0;
            long totalJobMemoryWasted = 0;
            for (AppResult job : jobDefIdToJobsMap.get(jobDefPair)) {
                totalJobMemoryUsed += job.resourceUsed;
                totalJobMemoryWasted += job.resourceWasted;
                if (job.severity.getValue() > jobseverity.getValue()) {
                    jobseverity = job.severity;
                }
            }
            if (jobSeverityCount.containsKey(jobseverity)) {
                jobSeverityCount.put(jobseverity, jobSeverityCount.get(jobseverity) + 1);
            } else {
                jobSeverityCount.put(jobseverity, 1L);
            }
            if (jobseverity.getValue() > flowSeverity.getValue()) {
                flowSeverity = jobseverity;
            }
            totalFlowMemoryUsed += totalJobMemoryUsed;
            totalFlowMemoryWasted += totalJobMemoryWasted;
        }
        totalFlowDelay = Utils.getTotalWaittime(flowExecIdToJobsMap.get(flowExecPair));
        totalFlowRuntime = Utils.getTotalRuntime(flowExecIdToJobsMap.get(flowExecPair));
        JsonArray jobSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(jobSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, jobSeverityCount.get(key));
            jobSeverity.add(severityObject);
        }
        // Execution record
        JsonObject dataset = new JsonObject();
        dataset.addProperty(JsonKeys.ID, mrJobsList.get(0).flowExecId);
        dataset.addProperty(JsonKeys.USERNAME, mrJobsList.get(0).username);
        dataset.addProperty(JsonKeys.START_TIME, mrJobsList.get(0).startTime);
        dataset.addProperty(JsonKeys.FINISH_TIME, mrJobsList.get(mrJobsList.size() - 1).finishTime);
        dataset.addProperty(JsonKeys.RUNTIME, totalFlowRuntime);
        dataset.addProperty(JsonKeys.WAITTIME, totalFlowDelay);
        dataset.addProperty(JsonKeys.RESOURCE_USED, totalFlowMemoryUsed);
        dataset.addProperty(JsonKeys.RESOURCE_WASTED, totalFlowMemoryWasted);
        dataset.addProperty(JsonKeys.QUEUE, mrJobsList.get(0).queueName);
        dataset.addProperty(JsonKeys.SEVERITY, flowSeverity.getText());
        dataset.addProperty(JsonKeys.SCHEDULER, mrJobsList.get(0).scheduler);
        dataset.addProperty(JsonKeys.FLOW_EXEC_ID, mrJobsList.get(0).flowExecId);
        dataset.addProperty(JsonKeys.FLOW_DEF_ID, mrJobsList.get(0).flowDefId);
        dataset.add(JsonKeys.JOBS_SEVERITY, jobSeverity);
        workflowSummaryArray.add(dataset);
    }
    JsonArray sortedWorkflowSummaryArray = getSortedJsonArrayByFinishTime(workflowSummaryArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.WORKFLOW_SUMMARIES, sortedWorkflowSummaryArray);
    return ok(new Gson().toJson(parent));
}

Example 3 with IdUrlPair

use of controllers.IdUrlPair in project dr-elephant by linkedin.

the class Web method restJobSummariesForUser.

/**
 * This method returns the json object for job-summaries for the given user
 * @param username The given username for which job-summaries json object should be returned
 * @return The job-summaries json object for the given username
 * response object:
 * <pre>
 *{
 *  "job-summaries": [
 *  {
 *    "id": "job-exec-id",
 *      "jobname": "jobname",
 *      "jobtype": "Pig",
 *      "username": "username",
 *      "starttime": 1471910835628,
 *      "finishtime": 1471911099238,
 *      "runtime": 263610,
 *      "waittime": 46234,
 *      "resourceused": 101382144,
 *      "resourcewasted": 15993417,
 *      "severity": "Moderate",
 *      "scheduler": "azkaban",
 *      "tasksseverity": [
 *    {
 *      "severity": "Moderate",
 *        "count": 1
 *    }
 *    ]
 *  }
 *  ]
 *}
 * </pre>
 */
public static Result restJobSummariesForUser(String username) {
    JsonArray jobSummaryArray = new JsonArray();
    List<AppResult> results = null;
    if (username == null || username.isEmpty()) {
        results = getSchedulerApplications(MAX_APPLICATIONS_IN_WORKFLOW);
    } else {
        results = getSchedulerApplications(username, MAX_APPLICATIONS_IN_WORKFLOW);
    }
    Map<IdUrlPair, List<AppResult>> jobExecIdToJobsMap = ControllerUtil.limitHistoryResults(ControllerUtil.groupJobs(results, ControllerUtil.GroupBy.JOB_EXECUTION_ID), results.size(), MAX_JOB_LIMIT);
    for (IdUrlPair jobDefPair : jobExecIdToJobsMap.keySet()) {
        long totalJobMemoryUsed = 0L;
        long totalJobMemoryWasted = 0L;
        long totalJobDelay = 0L;
        long totalJobRuntime = 0L;
        long jobStartTime = Long.MAX_VALUE;
        long jobEndTime = 0;
        Severity jobSeverity = Severity.NONE;
        String jobType = null;
        String jobId = jobDefPair.getId();
        String jobName = "";
        String user = null;
        String queueName = "";
        String scheduler = "";
        String jobDefId = "";
        String jobExecId = "";
        Map<Severity, Long> applicationSeverityCount = new HashMap<Severity, Long>();
        for (AppResult application : jobExecIdToJobsMap.get(jobDefPair)) {
            totalJobMemoryUsed += application.resourceUsed;
            totalJobMemoryWasted += application.resourceWasted;
            jobType = application.jobType;
            jobName = application.jobName;
            jobDefId = application.jobDefId;
            jobExecId = application.jobExecId;
            queueName = application.queueName;
            scheduler = application.scheduler;
            if (application.startTime < jobStartTime) {
                jobStartTime = application.startTime;
            }
            if (application.finishTime > jobEndTime) {
                jobEndTime = application.finishTime;
            }
            if (application.severity.getValue() > jobSeverity.getValue()) {
                jobSeverity = application.severity;
            }
            if (applicationSeverityCount.containsKey(application.severity)) {
                applicationSeverityCount.put(application.severity, applicationSeverityCount.get(application.severity) + 1L);
            } else {
                applicationSeverityCount.put(application.severity, 1L);
            }
            user = application.username;
        }
        JsonArray applicationSeverity = new JsonArray();
        List<Severity> keys = getSortedSeverityKeys(applicationSeverityCount.keySet());
        for (Severity key : keys) {
            JsonObject severityObject = new JsonObject();
            severityObject.addProperty(JsonKeys.SEVERITY, key.getText());
            severityObject.addProperty(JsonKeys.COUNT, applicationSeverityCount.get(key));
            applicationSeverity.add(severityObject);
        }
        totalJobDelay = Utils.getTotalWaittime(jobExecIdToJobsMap.get(jobDefPair));
        totalJobRuntime = Utils.getTotalRuntime(jobExecIdToJobsMap.get(jobDefPair));
        JsonObject jobObject = new JsonObject();
        jobObject.addProperty(JsonKeys.ID, jobId);
        jobObject.addProperty(JsonKeys.JOB_NAME, jobName);
        jobObject.addProperty(JsonKeys.JOB_TYPE, jobType);
        jobObject.addProperty(JsonKeys.USERNAME, user);
        jobObject.addProperty(JsonKeys.START_TIME, jobStartTime);
        jobObject.addProperty(JsonKeys.FINISH_TIME, jobEndTime);
        jobObject.addProperty(JsonKeys.RUNTIME, totalJobRuntime);
        jobObject.addProperty(JsonKeys.WAITTIME, totalJobDelay);
        jobObject.addProperty(JsonKeys.RESOURCE_USED, totalJobMemoryUsed);
        jobObject.addProperty(JsonKeys.RESOURCE_WASTED, totalJobMemoryWasted);
        jobObject.addProperty(JsonKeys.QUEUE, queueName);
        jobObject.addProperty(JsonKeys.SCHEDULER, scheduler);
        jobObject.addProperty(JsonKeys.SEVERITY, jobSeverity.getText());
        jobObject.addProperty(JsonKeys.JOB_DEF_ID, jobDefId);
        jobObject.addProperty(JsonKeys.JOB_EXEC_ID, jobExecId);
        jobObject.add(JsonKeys.TASKS_SEVERITY, applicationSeverity);
        jobSummaryArray.add(jobObject);
    }
    JsonArray sortedJobSummaryArray = getSortedJsonArrayByFinishTime(jobSummaryArray);
    JsonObject parent = new JsonObject();
    parent.add(JsonKeys.JOB_SUMMARIES, sortedJobSummaryArray);
    return ok(new Gson().toJson(parent));
}