Examples with SchedulerConfigurationData com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData used on opensource projects

Example 6 with SchedulerConfigurationData

use of com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData in project dr-elephant by linkedin.

the class SchedulerConfigurationTest method testParseSchedulerConf1.

/**
 * Correctly configured scheduler
 */
@Test
public void testParseSchedulerConf1() {
    SchedulerConfiguration schedulerConf = new SchedulerConfiguration(document1.getDocumentElement());
    List<SchedulerConfigurationData> schedulerConfData = schedulerConf.getSchedulerConfigurationData();
    assertEquals(schedulerConfData.size(), 2);
    for (SchedulerConfigurationData data : schedulerConfData) {
        if (data.getSchedulerName().equals("airflow")) {
            assertEquals("com.linkedin.drelephant.schedulers.AirflowScheduler", data.getClassName());
            assertEquals("http://localhost:8000", data.getParamMap().get("airflowbaseurl"));
        } else {
            assertEquals("azkaban", data.getSchedulerName());
            assertEquals("com.linkedin.drelephant.schedulers.AzkabanScheduler", data.getClassName());
        }
    }
}

Example 7 with SchedulerConfigurationData

use of com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData in project dr-elephant by linkedin.

the class InfoExtractorTest method testLoadSchedulerInfo.

@Test
public void testLoadSchedulerInfo() {
    Properties properties = new Properties();
    properties.put(AzkabanScheduler.AZKABAN_JOB_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name");
    properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0");
    properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, "https://grid.example.com:9000/manager?project=project-name&flow=flow-name");
    properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, "https://grid.example.com:9000/executor?execid=123456");
    properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, "job-name");
    SchedulerConfigurationData schedulerConfigurationData = new SchedulerConfigurationData("azkaban", null, null);
    Scheduler scheduler = new AzkabanScheduler("id", properties, schedulerConfigurationData);
    AppResult result = new AppResult();
    HadoopApplicationData data = new HadoopApplicationData() {

        String appId = "application_5678";

        Properties conf = new Properties();

        ApplicationType applicationType = new ApplicationType("foo");

        @Override
        public String getAppId() {
            return appId;
        }

        @Override
        public Properties getConf() {
            return conf;
        }

        @Override
        public ApplicationType getApplicationType() {
            return applicationType;
        }

        @Override
        public boolean isEmpty() {
            return false;
        }
    };
    InfoExtractor.loadSchedulerInfo(result, data, scheduler);
    assertEquals(result.scheduler, "azkaban");
    assertFalse(StringUtils.isEmpty(result.getJobExecId()));
    assertFalse(StringUtils.isEmpty(result.getJobDefId()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecId()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefId()));
    assertFalse(StringUtils.isEmpty(result.getJobExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getJobDefUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowExecUrl()));
    assertFalse(StringUtils.isEmpty(result.getFlowDefUrl()));
}

Example 8 with SchedulerConfigurationData

use of com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData in project dr-elephant by linkedin.

the class Utils method getFlowTime.

/**
 * Get the flowtime corresponding to the job.
 * The flowtime will be used in the front-end as the entries under the 'Flow Executions' column (in the Flow/Job history view).
 *
 * By default, the finishTime value of the job is used. However, this can be configured in the scheduler configuration,
 * so that another field is used instead, with the appropriate formatting to convert it to a long value.
 *
 * This method can be extended to include other flowtimefield and flowtimetype.
 */
public static long getFlowTime(AppResult result) {
    String schedulerName = result.scheduler;
    SchedulerConfigurationData schedulerData = InfoExtractor.getSchedulerData(schedulerName);
    String flowTimeField = null;
    String flowTimeType = null;
    if (schedulerData != null) {
        flowTimeField = schedulerData.getParamMap().get("flowtimefield");
        flowTimeType = schedulerData.getParamMap().get("flowtimetype");
    }
    if (flowTimeField != null && flowTimeType != null && flowTimeField.equals("flowExecId")) {
        SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(flowTimeType);
        long flowTime;
        try {
            flowTime = DATE_FORMAT.parse(result.flowExecId).getTime();
        } catch (ParseException e) {
            logger.warn("Could not parse " + result.flowExecId + " for application " + result.id);
            flowTime = result.finishTime;
        }
        return flowTime;
    } else {
        return result.finishTime;
    }
}

Example 9 with SchedulerConfigurationData

use of com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData in project dr-elephant by linkedin.

the class OozieSchedulerTest method testManualCommittedJobAppNameUnique.

/*
    Job Reference ID: Job AppName-ActionName
    Job Execution ID: Oozie Job ID
    Flow Reference ID: Super Parent Job AppName
    Flow Execution ID: Super Parent Oozie job ID
     */
@Test
public void testManualCommittedJobAppNameUnique() throws Exception {
    Map<String, String> params = new HashMap<String, String>();
    params.put("oozie_app_name_uniqueness", "true");
    SchedulerConfigurationData schedulerConfig = makeConfig("oozie", params);
    OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, manualCommittedJobClient);
    assertEquals(oozieUrl + jobInfo, scheduler.getJobDefUrl());
    assertEquals(oozieUrl + jobInfo, scheduler.getJobExecUrl());
    assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowDefUrl());
    assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowExecUrl());
    assertEquals(1, scheduler.getWorkflowDepth());
    assertEquals(jobAppName + "-" + actionName, scheduler.getJobName());
    assertEquals("oozie", scheduler.getSchedulerName());
}

Example 10 with SchedulerConfigurationData

use of com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData in project dr-elephant by linkedin.

the class OozieSchedulerTest method testDepthCalculation.

@Test
public void testDepthCalculation() throws Exception {
    when(workflowJob.getParentId()).thenReturn(jobInfo, jobInfo, jobInfo, null);
    SchedulerConfigurationData schedulerConfig = makeConfig("oozie", new HashMap<String, String>());
    OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, oozieClient);
    assertEquals(1, scheduler.getWorkflowDepth());
}