Examples with TuningJobExecution models.TuningJobExecution used on opensource projects

Example 6 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class RestAPITest method testRestGetCurrentRunParameters.

@Test
public void testRestGetCurrentRunParameters() {
    Configuration configuration = ElephantContext.instance().getAutoTuningConf();
    Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false);
    org.junit.Assume.assumeTrue(autoTuningEnabled);
    running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() {

        public void run() {
            populateAutoTuningTestData1();
            JsonNode jsonNode = getTestGetCurrentRunParameterData();
            final WS.Response response = WS.url(BASE_URL + REST_GET_CURRENT_RUN_PARAMETERS).post(jsonNode).get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
            final JsonNode jsonResponse = response.asJson();
            logger.info("Output of getCurrentRunParameters ");
            logger.info(jsonResponse.toString());
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.map.memory.mb").asDouble() > 0);
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() > 0);
            assertTrue("Get current run param output size did not match", jsonResponse.size() == 9);
            TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*").fetch(TuningJobExecution.TABLE.jobExecution, "*").fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0").findUnique();
            tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED;
            tuningJobExecution.jobExecution.executionState = ExecutionState.SUCCEEDED;
            tuningJobExecution.update();
            FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil();
            fitnessComputeUtil.updateFitness();
            tuningJobExecution = TuningJobExecution.find.select("*").fetch(TuningJobExecution.TABLE.jobExecution, "*").fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0").findUnique();
            assertTrue("Fitness not computed", tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED);
            assertTrue("Fitness not computed and have zero value", tuningJobExecution.fitness > 0);
        }
    });
}

Example 7 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AutoTuningAPIHelper method getCurrentRunParameters.

/**
 * Handles the api request and returns param suggestions as response
 * @param tuningInput Rest api parameters
 * @return Parameter Suggestion
 */
public Map<String, Double> getCurrentRunParameters(TuningInput tuningInput) {
    logger.info("Parameter suggestion request for execution: " + tuningInput.getJobExecId());
    setDefaultValue(tuningInput);
    String jobDefId = tuningInput.getJobDefId();
    TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").fetch(TuningJobDefinition.TABLE.job, "*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId).eq(TuningJobDefinition.TABLE.tuningEnabled, 1).findUnique();
    // If new job for tuning, update db with new job configuration
    if (tuningJobDefinition == null) {
        logger.debug("New job encountered for tuning");
        AutoTuningMetricsController.markNewAutoTuningJob();
        tuningJobDefinition = addNewJobForTuning(tuningInput);
    }
    logger.debug("Finding parameter suggestion for job: " + tuningJobDefinition.job.jobName);
    TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*").fetch(TuningJobExecution.TABLE.jobExecution, "*").fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id).eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.CREATED).order().asc(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.createdTs).setMaxRows(1).findUnique();
    // If no new parameter set for suggestion, create a new suggestion with default parameter
    if (tuningJobExecution == null) {
        logger.info("Returning default parameters as no parameter suggestion found for job: " + tuningJobDefinition.job.jobName);
        AutoTuningMetricsController.markParamSetNotFound();
        tuningJobExecution = createDefaultJobExecution(tuningJobDefinition);
    }
    logger.debug("Finding parameters corresponding to execution id: " + tuningJobExecution.jobExecution.id);
    List<JobSuggestedParamValue> jobSuggestedParamValues = JobSuggestedParamValue.find.where().eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, tuningJobExecution.jobExecution.id).findList();
    logger.debug("Number of output parameters : " + jobSuggestedParamValues.size());
    Map<String, Double> paramValues = new HashMap<String, Double>();
    if (jobSuggestedParamValues != null) {
        for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) {
            logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + jobSuggestedParamValue.paramValue);
            paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue);
        }
    }
    updateJobExecutionParameter(tuningJobExecution, tuningInput);
    logger.info("Finishing getCurrentRunParameters");
    return paramValues;
}

Example 8 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AutoTuningAPIHelper method updateJobExecutionParameter.

/**
 *This is to update job execution with IN_PROGRESS and parameter set with IN_PROGRESS. Also update flow_exec_id
 *, flowExecURL, JobExecID and jobExecURL
 * @param tuningJobExecution
 * @param tuningInput
 */
private void updateJobExecutionParameter(TuningJobExecution tuningJobExecution, TuningInput tuningInput) {
    FlowExecution flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, tuningInput.getFlowExecId()).findUnique();
    if (flowExecution == null) {
        flowExecution = new FlowExecution();
        flowExecution.flowExecId = tuningInput.getFlowExecId();
        flowExecution.flowExecUrl = tuningInput.getFlowExecUrl();
        flowExecution.flowDefinition = tuningJobExecution.jobExecution.job.flowDefinition;
        flowExecution.save();
    }
    JobExecution jobExecution = tuningJobExecution.jobExecution;
    jobExecution.jobExecId = tuningInput.getJobExecId();
    jobExecution.jobExecUrl = tuningInput.getJobExecUrl();
    jobExecution.executionState = ExecutionState.IN_PROGRESS;
    jobExecution.flowExecution = flowExecution;
    logger.debug("Saving job execution" + jobExecution.jobExecId);
    jobExecution.save();
    tuningJobExecution.jobExecution = jobExecution;
    tuningJobExecution.paramSetState = ParamSetStatus.SENT;
    tuningJobExecution.save();
}

Example 9 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class AzkabanJobCompleteDetector method getCompletedExecutions.

/**
 * Returns the list of completed executions
 * @param jobExecutions Started Execution list
 * @return List of completed executions
 * @throws MalformedURLException
 * @throws URISyntaxException
 */
protected List<TuningJobExecution> getCompletedExecutions(List<TuningJobExecution> jobExecutions) throws MalformedURLException, URISyntaxException {
    logger.info("Fetching the list of executions completed since last iteration");
    List<TuningJobExecution> completedExecutions = new ArrayList<TuningJobExecution>();
    try {
        for (TuningJobExecution tuningJobExecution : jobExecutions) {
            JobExecution jobExecution = tuningJobExecution.jobExecution;
            logger.info("Checking current status of started execution: " + tuningJobExecution.jobExecution.jobExecId);
            if (_azkabanJobStatusUtil == null) {
                logger.info("Initializing  AzkabanJobStatusUtil");
                _azkabanJobStatusUtil = new AzkabanJobStatusUtil();
            }
            try {
                Map<String, String> jobStatus = _azkabanJobStatusUtil.getJobsFromFlow(jobExecution.flowExecution.flowExecId);
                if (jobStatus != null) {
                    for (Map.Entry<String, String> job : jobStatus.entrySet()) {
                        logger.info("Job Found:" + job.getKey() + ". Status: " + job.getValue());
                        if (job.getKey().equals(jobExecution.job.jobName)) {
                            if (job.getValue().equals(AzkabanJobStatus.FAILED.toString())) {
                                tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED;
                                jobExecution.executionState = ExecutionState.FAILED;
                            }
                            if (job.getValue().equals(AzkabanJobStatus.CANCELLED.toString()) || job.getValue().equals(AzkabanJobStatus.KILLED.toString())) {
                                tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED;
                                jobExecution.executionState = ExecutionState.CANCELLED;
                            }
                            if (job.getValue().equals(AzkabanJobStatus.SUCCEEDED.toString())) {
                                tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED;
                                jobExecution.executionState = ExecutionState.SUCCEEDED;
                            }
                            if (tuningJobExecution.paramSetState.equals(ParamSetStatus.EXECUTED)) {
                                completedExecutions.add(tuningJobExecution);
                                logger.info("Execution " + tuningJobExecution.jobExecution.jobExecId + " is completed");
                            } else {
                                logger.info("Execution " + tuningJobExecution.jobExecution.jobExecId + " is still in running state");
                            }
                        }
                    }
                } else {
                    logger.info("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId);
                }
            } catch (Exception e) {
                logger.error("Error in checking status of execution: " + jobExecution.jobExecId, e);
            }
        }
    } catch (Exception e) {
        logger.error("Error in fetching list of completed executions", e);
        e.printStackTrace();
    }
    logger.info("Number of executions completed since last iteration: " + completedExecutions.size());
    return completedExecutions;
}

Example 10 with TuningJobExecution

use of models.TuningJobExecution in project dr-elephant by linkedin.

the class FitnessComputeUtil method updateMetrics.

/**
 * This method update metrics for auto tuning monitoring for fitness compute daemon
 * @param completedExecutions
 */
private void updateMetrics(List<TuningJobExecution> completedExecutions) {
    int fitnessNotUpdated = 0;
    for (TuningJobExecution tuningJobExecution : completedExecutions) {
        if (tuningJobExecution.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED) == false) {
            fitnessNotUpdated++;
        } else {
            AutoTuningMetricsController.markFitnessComputedJobs();
        }
    }
    AutoTuningMetricsController.setFitnessComputeWaitJobs(fitnessNotUpdated);
}