Examples with Scheduler com.linkedin.drelephant.schedulers.Scheduler used on opensource projects

Example 6 with Scheduler

use of com.linkedin.drelephant.schedulers.Scheduler in project dr-elephant by linkedin.

the class InfoExtractorTest method testLoadSchedulerInfo.

@Test
public void testLoadSchedulerInfo() {
    Properties properties = new Properties();
    properties.put(AzkabanScheduler.AZKABAN_JOB_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name");
    properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0");
    properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name");
    properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, "https://grid.example.com:9000/executor?execid=123456");
    properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, "job-name");
    SchedulerConfigurationData schedulerConfigurationData = new SchedulerConfigurationData("azkaban", null, null);
    Scheduler scheduler = new AzkabanScheduler("id", properties, schedulerConfigurationData);
    AppResult result = new AppResult();
    HadoopApplicationData data = new HadoopApplicationData() {

        String appId = "application_5678";

        Properties conf = new Properties();

        ApplicationType applicationType = new ApplicationType("foo");

        @Override
        public String getAppId() {
            return appId;
        }

        @Override
        public Properties getConf() {
            return conf;
        }

        @Override
        public ApplicationType getApplicationType() {
            return applicationType;
        }

        @Override
        public boolean isEmpty() {
            return false;
        }
    };
    InfoExtractor.loadSchedulerInfo(result, data, scheduler);
    assertEquals(result.scheduler, "azkaban");
    assertFalse(StringUtils.isEmpty(result.getJobExecId()));
    assertFalse(StringUtils.isEmpty(result.getJobDefId()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecId()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefId()));
    assertFalse(StringUtils.isEmpty(result.getJobExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getJobDefUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefUrl()));
}

Example 7 with Scheduler

use of com.linkedin.drelephant.schedulers.Scheduler in project dr-elephant by linkedin.

the class InfoExtractor method loadInfo.

/**
 * Loads result with the info depending on the application type
 *
 * @param result The jobResult to be loaded with.
 * @param data The Hadoop application data
 */
public static void loadInfo(AppResult result, HadoopApplicationData data) {
    Properties properties = new Properties();
    if (data instanceof MapReduceApplicationData) {
        properties = retrieveMapreduceProperties((MapReduceApplicationData) data);
    } else if (data instanceof SparkApplicationData) {
        properties = retrieveSparkProperties((SparkApplicationData) data);
    } else if (data instanceof TezApplicationData) {
        properties = retrieveTezProperties((TezApplicationData) data);
    }
    Scheduler scheduler = getSchedulerInstance(data.getAppId(), properties);
    if (scheduler == null) {
        logger.info("No Scheduler found for appid: " + data.getAppId());
        loadNoSchedulerInfo(result);
    } else if (StringUtils.isEmpty(scheduler.getJobDefId()) || StringUtils.isEmpty(scheduler.getJobExecId()) || StringUtils.isEmpty(scheduler.getFlowDefId()) || StringUtils.isEmpty(scheduler.getFlowExecId())) {
        logger.warn("This job doesn't have the correct " + scheduler.getSchedulerName() + " integration support. I" + " will treat this as an adhoc job");
        logger.info("No Flow/job info found for appid: " + data.getAppId());
        loadNoSchedulerInfo(result);
    } else {
        loadSchedulerInfo(result, data, scheduler);
    }
}