Examples with BaselineComputeUtil com.linkedin.drelephant.tuning.BaselineComputeUtil used on opensource projects

Example 1 with BaselineComputeUtil

use of com.linkedin.drelephant.tuning.BaselineComputeUtil in project dr-elephant by linkedin.

the class AutoTuner method run.

public void run() {
    logger.info("Starting Auto Tuning thread");
    HDFSContext.load();
    Configuration configuration = ElephantContext.instance().getAutoTuningConf();
    Long interval = Utils.getNonNegativeLong(configuration, AUTO_TUNING_DAEMON_WAIT_INTERVAL, DEFAULT_METRICS_COMPUTATION_INTERVAL);
    try {
        AutoTuningMetricsController.init();
        BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil();
        FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil();
        ParamGenerator paramGenerator = new PSOParamGenerator();
        JobCompleteDetector jobCompleteDetector = new AzkabanJobCompleteDetector();
        while (!Thread.currentThread().isInterrupted()) {
            try {
                baselineComputeUtil.computeBaseline();
                jobCompleteDetector.updateCompletedExecutions();
                fitnessComputeUtil.updateFitness();
                paramGenerator.getParams();
            } catch (Exception e) {
                logger.error("Error in auto tuner thread ", e);
            }
            Thread.sleep(interval);
        }
    } catch (Exception e) {
        logger.error("Error in auto tuner thread ", e);
    }
    logger.info("Auto tuning thread shutting down");
}

Example 2 with BaselineComputeUtil

use of com.linkedin.drelephant.tuning.BaselineComputeUtil in project dr-elephant by linkedin.

the class RestAPITest method testRestGetCurrentRunParametersNewJob.

@Test
public void testRestGetCurrentRunParametersNewJob() {
    Configuration configuration = ElephantContext.instance().getAutoTuningConf();
    Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false);
    org.junit.Assume.assumeTrue(autoTuningEnabled);
    running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() {

        public void run() {
            populateAutoTuningTestData1();
            JsonNode jsonNode = getTestGetCurrentRunParameterNewData();
            final WS.Response response = WS.url(BASE_URL + REST_GET_CURRENT_RUN_PARAMETERS).post(jsonNode).get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
            final JsonNode jsonResponse = response.asJson();
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.map.memory.mb").asDouble() == 2048D);
            assertTrue("Get current run param output did not match", jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() == 2048D);
            assertTrue("Get current run param output size did not match", jsonResponse.size() == 2);
            TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry").findUnique();
            assertTrue("New Job Not created  ", tuningJobDefinition.job.jobName.equals("countByCountryFlowSmallNew_countByCountry"));
            BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil();
            baselineComputeUtil.computeBaseline();
            tuningJobDefinition = TuningJobDefinition.find.select("*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry").findUnique();
            assertTrue("Baseline not computed:averageResourceUsage  ", tuningJobDefinition.averageResourceUsage > 0);
            assertTrue("Baseline not computed:averageInputSizeInBytes  ", tuningJobDefinition.averageInputSizeInBytes > 0);
            assertTrue("Baseline not computed:averageExecutionTime  ", tuningJobDefinition.averageExecutionTime > 0);
        }
    });
}