Examples with TuningJobExecution models.TuningJobExecution used on opensource projects

Example 1 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class JobCompleteDetector method updateExecutionStatus.

/**
 * Updates the job execution status
 * @param jobExecutions JobExecution list
 * @return Update status
 */
private void updateExecutionStatus(List<TuningJobExecution> jobExecutions) {
    logger.info("Updating status of executions completed since last iteration");
    for (TuningJobExecution tuningJobExecution : jobExecutions) {
        JobExecution jobExecution = tuningJobExecution.jobExecution;
        logger.info("Updating execution status to EXECUTED for the execution: " + jobExecution.jobExecId);
        jobExecution.update();
        tuningJobExecution.update();
    }
}

Example 2 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class APIFitnessComputeUtil method updateExecutionMetrics.

/**
 * Updates the execution metrics
 * @param completedExecutions List of completed executions
 */
protected void updateExecutionMetrics(List<TuningJobExecution> completedExecutions) {
    logger.debug("Updating execution metrics");
    updateAuthToken();
    for (TuningJobExecution tuningJobExecution : completedExecutions) {
        logger.debug("Completed executions before updating metric: " + Json.toJson(tuningJobExecution));
        try {
            JobExecution jobExecution = tuningJobExecution.jobExecution;
            JobDefinition job = jobExecution.job;
            URL jobExecURL = new URL(new URL(_drElephantURL), String.format("/rest/jobexec?id=%s", URLEncoder.encode(jobExecution.jobExecId)));
            HttpURLConnection conn = (HttpURLConnection) jobExecURL.openConnection();
            JsonNode allApps = _objectMapper.readTree(conn.getInputStream());
            // job id match and tuning enabled
            TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").fetch(TuningJobDefinition.TABLE.job, "*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id).eq(TuningJobDefinition.TABLE.tuningEnabled, 1).findUnique();
            if (allApps != null && allApps.size() > 0) {
                Long totalExecutionTime = 0L;
                Double totalResourceUsed = 0D;
                Double totalInputBytesInBytes = 0D;
                for (JsonNode app : allApps) {
                    logger.info("Job Execution Update: ApplicationID " + app.get("id").getTextValue());
                    Long executionTime = app.get("finishTime").getLongValue() - app.get("startTime").getLongValue() - app.get("totalDelay").getLongValue();
                    totalExecutionTime += executionTime;
                    totalResourceUsed += app.get("resourceUsed").getDoubleValue();
                    totalInputBytesInBytes += getTotalInputBytes(app.get("id").getTextValue());
                }
                if (totalExecutionTime != 0) {
                    jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60);
                    jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600);
                    jobExecution.inputSizeInBytes = totalInputBytesInBytes;
                    logger.info("Job Execution Update: UpdatedValue " + totalExecutionTime + ":" + totalResourceUsed + ":" + totalInputBytesInBytes);
                }
                logger.debug("Job execution " + jobExecution.resourceUsage);
                logger.debug("Job details: AvgResourceUsage " + tuningJobDefinition.averageResourceUsage + ", allowedMaxResourceUsagePercent: " + tuningJobDefinition.allowedMaxResourceUsagePercent);
                if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals(JobExecution.ExecutionState.CANCELLED)) {
                    // Todo: Check if the reason of failure is auto tuning and  handle cancelled cases
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * tuningJobDefinition.averageInputSizeInBytes);
                } else if (jobExecution.resourceUsage > (tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0)) {
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * totalInputBytesInBytes);
                } else {
                    tuningJobExecution.fitness = jobExecution.resourceUsage * FileUtils.ONE_GB / totalInputBytesInBytes;
                }
                tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED;
                jobExecution.update();
                tuningJobExecution.update();
                logger.debug("Completed executions after updating metrics: " + Json.toJson(tuningJobExecution));
            } else {
                if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals(JobExecution.ExecutionState.CANCELLED)) {
                    // Todo: Check if the reason of failure is auto tuning and  handle cancelled cases
                    tuningJobExecution.fitness = 3 * tuningJobDefinition.averageResourceUsage * tuningJobDefinition.allowedMaxResourceUsagePercent * FileUtils.ONE_GB / (100.0 * tuningJobDefinition.averageInputSizeInBytes);
                    jobExecution.executionTime = 0D;
                    jobExecution.resourceUsage = 0D;
                    jobExecution.inputSizeInBytes = 0D;
                    tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED;
                    jobExecution.update();
                    tuningJobExecution.update();
                }
            }
        } catch (Exception e) {
            logger.error("Error updating fitness of job_exec_id: " + tuningJobExecution.jobExecution.id + "\n Stacktrace: ", e);
        }
    }
    logger.debug("Execution metrics updated");
}

Example 3 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AutoTuningAPIHelper method insertDefaultJobExecution.

/**
 * Inserts default job execution in database
 * @param job Job
 * @param flowExecId Flow execution id
 * @param jobExecId Job execution id
 * @param flowExecUrl Flow execution url
 * @param jobExecUrl Job execution url
 * @return default job execution
 */
private TuningJobExecution insertDefaultJobExecution(JobDefinition job, String flowExecId, String jobExecId, String flowExecUrl, String jobExecUrl, FlowDefinition flowDefinition, TuningAlgorithm tuningAlgorithm) {
    logger.debug("Starting insertDefaultJobExecution");
    FlowExecution flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, flowExecId).findUnique();
    if (flowExecution == null) {
        flowExecution = new FlowExecution();
        flowExecution.flowExecId = flowExecId;
        flowExecution.flowExecUrl = flowExecUrl;
        flowExecution.flowDefinition = flowDefinition;
        flowExecution.save();
    }
    JobExecution jobExecution = JobExecution.find.where().eq(JobExecution.TABLE.jobExecId, jobExecId).findUnique();
    if (jobExecution == null) {
        jobExecution = new JobExecution();
        jobExecution.job = job;
        jobExecution.executionState = ExecutionState.NOT_STARTED;
        jobExecution.jobExecId = jobExecId;
        jobExecution.jobExecUrl = jobExecUrl;
        jobExecution.flowExecution = flowExecution;
        jobExecution.save();
    }
    TuningJobExecution tuningJobExecution = new TuningJobExecution();
    tuningJobExecution.jobExecution = jobExecution;
    tuningJobExecution.tuningAlgorithm = tuningAlgorithm;
    tuningJobExecution.paramSetState = ParamSetStatus.CREATED;
    tuningJobExecution.isDefaultExecution = true;
    tuningJobExecution.save();
    logger.debug("Finishing insertDefaultJobExecution. Job Execution ID " + jobExecution.jobExecId);
    return tuningJobExecution;
}

Example 4 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AutoTuningAPIHelper method createDefaultJobExecution.

/**
 * This method creates a job execution with default parameters. This is required when there is no parameters set
 * remains for suggestion.
 * @param tuningJobDefinition Job definition
 * @return
 */
private TuningJobExecution createDefaultJobExecution(TuningJobDefinition tuningJobDefinition) {
    logger.info("Creating an execution with default parameter values for job: " + tuningJobDefinition.job.jobDefId);
    // Get default execution from DB and clone that to create a new default execution
    TuningJobExecution tuningJobExecutionDefault = TuningJobExecution.find.select("*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id).eq(TuningJobExecution.TABLE.isDefaultExecution, true).setMaxRows(1).findUnique();
    TuningJobExecution tuningJobExecution = new TuningJobExecution();
    JobExecution jobExecution = new JobExecution();
    jobExecution.id = 0L;
    jobExecution.job = tuningJobExecutionDefault.jobExecution.job;
    jobExecution.executionState = ExecutionState.NOT_STARTED;
    jobExecution.save();
    tuningJobExecution.jobExecution = jobExecution;
    tuningJobExecution.isDefaultExecution = tuningJobExecutionDefault.isDefaultExecution;
    tuningJobExecution.tuningAlgorithm = tuningJobExecutionDefault.tuningAlgorithm;
    tuningJobExecution.paramSetState = ParamSetStatus.CREATED;
    tuningJobExecution.save();
    logger.debug("Execution with default parameter created with execution id: " + tuningJobExecution.jobExecution.id);
    List<JobSuggestedParamValue> jobSuggestedParamValueList = JobSuggestedParamValue.find.where().eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, tuningJobExecutionDefault.jobExecution.id).findList();
    // Save default parameters corresponding to new default execution
    for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) {
        JobSuggestedParamValue jobSuggestedParamValue1 = new JobSuggestedParamValue();
        jobSuggestedParamValue1.id = 0;
        jobSuggestedParamValue1.jobExecution = jobExecution;
        jobSuggestedParamValue1.paramValue = jobSuggestedParamValue.paramValue;
        jobSuggestedParamValue1.tuningParameter = jobSuggestedParamValue.tuningParameter;
        jobSuggestedParamValue1.save();
    }
    tuningJobExecution = TuningJobExecution.find.select("*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id, tuningJobExecution.jobExecution.id).setMaxRows(1).findUnique();
    return tuningJobExecution;
}

Example 5 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AutoTuningAPIHelper method addNewJobForTuning.

/**
 * Add new job for tuning
 * @param tuningInput Tuning input parameters
 * @return Job
 */
private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) {
    logger.info("Adding new job for tuning, job id: " + tuningInput.getJobDefId());
    JobDefinition job = JobDefinition.find.select("*").where().eq(JobDefinition.TABLE.jobDefId, tuningInput.getJobDefId()).findUnique();
    FlowDefinition flowDefinition = FlowDefinition.find.where().eq(FlowDefinition.TABLE.flowDefId, tuningInput.getFlowDefId()).findUnique();
    if (flowDefinition == null) {
        flowDefinition = new FlowDefinition();
        flowDefinition.flowDefId = tuningInput.getFlowDefId();
        flowDefinition.flowDefUrl = tuningInput.getFlowDefUrl();
        flowDefinition.save();
    }
    if (job == null) {
        job = new JobDefinition();
        job.jobDefId = tuningInput.getJobDefId();
        job.scheduler = tuningInput.getScheduler();
        job.username = tuningInput.getUserName();
        job.jobName = tuningInput.getJobName();
        job.jobDefUrl = tuningInput.getJobDefUrl();
        job.flowDefinition = flowDefinition;
        job.save();
    }
    String flowExecId = tuningInput.getFlowExecId();
    String jobExecId = tuningInput.getJobExecId();
    String flowExecUrl = tuningInput.getFlowExecUrl();
    String jobExecUrl = tuningInput.getJobExecUrl();
    String client = tuningInput.getClient();
    String defaultParams = tuningInput.getDefaultParams();
    TuningJobDefinition tuningJobDefinition = new TuningJobDefinition();
    tuningJobDefinition.job = job;
    tuningJobDefinition.client = client;
    tuningJobDefinition.tuningAlgorithm = tuningInput.getTuningAlgorithm();
    tuningJobDefinition.tuningEnabled = 1;
    tuningJobDefinition.allowedMaxExecutionTimePercent = tuningInput.getAllowedMaxExecutionTimePercent();
    tuningJobDefinition.allowedMaxResourceUsagePercent = tuningInput.getAllowedMaxResourceUsagePercent();
    tuningJobDefinition.save();
    TuningJobExecution tuningJobExecution = insertDefaultJobExecution(job, flowExecId, jobExecId, flowExecUrl, jobExecUrl, flowDefinition, tuningInput.getTuningAlgorithm());
    insertDefaultParameters(tuningJobExecution.jobExecution, defaultParams);
    logger.info("Added job: " + tuningInput.getJobDefId() + " for tuning");
    return tuningJobDefinition;
}