Examples with FitnessComputeUtil com.linkedin.drelephant.tuning.FitnessComputeUtil used on opensource projects

Example 1 with FitnessComputeUtil

use of com.linkedin.drelephant.tuning.FitnessComputeUtil in project dr-elephant by linkedin.

the class AutoTuner method run.

public void run() {
    logger.info("Starting Auto Tuning thread");
    HDFSContext.load();
    Configuration configuration = ElephantContext.instance().getAutoTuningConf();
    Long interval = Utils.getNonNegativeLong(configuration, AUTO_TUNING_DAEMON_WAIT_INTERVAL, DEFAULT_METRICS_COMPUTATION_INTERVAL);
    try {
        AutoTuningMetricsController.init();
        BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil();
        FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil();
        ParamGenerator paramGenerator = new PSOParamGenerator();
        JobCompleteDetector jobCompleteDetector = new AzkabanJobCompleteDetector();
        while (!Thread.currentThread().isInterrupted()) {
            try {
                baselineComputeUtil.computeBaseline();
                jobCompleteDetector.updateCompletedExecutions();
                fitnessComputeUtil.updateFitness();
                paramGenerator.getParams();
            } catch (Exception e) {
                logger.error("Error in auto tuner thread ", e);
            }
            Thread.sleep(interval);
        }
    } catch (Exception e) {
        logger.error("Error in auto tuner thread ", e);
    }
    logger.info("Auto tuning thread shutting down");
}

Example 2 with FitnessComputeUtil

use of com.linkedin.drelephant.tuning.FitnessComputeUtil in project dr-elephant by linkedin.

the class RestAPITest method testRestGetCurrentRunParameters.

@Test
public void testRestGetCurrentRunParameters() {
    Configuration configuration = ElephantContext.instance().getAutoTuningConf();
    Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false);
    org.junit.Assume.assumeTrue(autoTuningEnabled);
    running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() {

        public void run() {
            populateAutoTuningTestData1();
            JsonNode jsonNode = getTestGetCurrentRunParameterData();
            final WS.Response response = WS.url(BASE_URL + REST_GET_CURRENT_RUN_PARAMETERS).post(jsonNode).get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
            final JsonNode jsonResponse = response.asJson();
            logger.info("Output of getCurrentRunParameters ");
            logger.info(jsonResponse.toString());
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.map.memory.mb").asDouble() > 0);
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() > 0);
            assertTrue("Get current run param output size did not match", jsonResponse.size() == 9);
            TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*").fetch(TuningJobExecution.TABLE.jobExecution, "*").fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0").findUnique();
            tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED;
            tuningJobExecution.jobExecution.executionState = ExecutionState.SUCCEEDED;
            tuningJobExecution.update();
            FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil();
            fitnessComputeUtil.updateFitness();
            tuningJobExecution = TuningJobExecution.find.select("*").fetch(TuningJobExecution.TABLE.jobExecution, "*").fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*").where().eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0").findUnique();
            assertTrue("Fitness not computed", tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED);
            assertTrue("Fitness not computed and have zero value", tuningJobExecution.fitness > 0);
        }
    });
}