Examples with MRJobLauncher org.apache.gobblin.runtime.mapreduce.MRJobLauncher used on opensource projects

Example 1 with MRJobLauncher

use of org.apache.gobblin.runtime.mapreduce.MRJobLauncher in project incubator-gobblin by apache.

the class TestJobLauncherExecutionDriver method testConstructor.

@Test
public void testConstructor() throws IOException, InterruptedException {
    File tmpTestDir = Files.createTempDir();
    try {
        File localJobRootDir = new File(tmpTestDir, "localJobRoot");
        Assert.assertTrue(localJobRootDir.mkdir());
        final Logger log = LoggerFactory.getLogger(getClass().getSimpleName() + ".testConstructor");
        Config jobConf1 = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob")).withValue(ConfigurationKeys.JOB_GROUP_KEY, ConfigValueFactory.fromAnyRef("myGroup")).withValue(ConfigurationKeys.JOB_DESCRIPTION_KEY, ConfigValueFactory.fromAnyRef("Awesome job")).withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(localJobRootDir.getPath())).withValue(ConfigurationKeys.SOURCE_CLASS_KEY, ConfigValueFactory.fromAnyRef(TestingSource.class.getName())).withValue(ConfigurationKeys.JOB_LOCK_ENABLED_KEY, ConfigValueFactory.fromAnyRef(false));
        JobSpec jobSpec1 = JobSpec.builder().withConfig(jobConf1).build();
        JobLauncherExecutionDriver.Launcher launcher = new JobLauncherExecutionDriver.Launcher().withJobLauncherType(JobLauncherFactory.JobLauncherType.LOCAL).withLog(log);
        JobLauncherExecutionDriver jled = (JobLauncherExecutionDriver) launcher.launchJob(jobSpec1);
        Assert.assertTrue(jled.getLegacyLauncher() instanceof LocalJobLauncher);
        JobExecution jex1 = jled.getJobExecution();
        Assert.assertEquals(jex1.getJobSpecURI(), jobSpec1.getUri());
        Assert.assertEquals(jex1.getJobSpecVersion(), jobSpec1.getVersion());
        Thread.sleep(5000);
        File mrJobRootDir = new File(tmpTestDir, "mrJobRoot");
        Assert.assertTrue(mrJobRootDir.mkdir());
        Config jobConf2 = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob2")).withValue(ConfigurationKeys.JOB_GROUP_KEY, ConfigValueFactory.fromAnyRef("myGroup")).withValue(ConfigurationKeys.JOB_DESCRIPTION_KEY, ConfigValueFactory.fromAnyRef("Awesome job")).withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(mrJobRootDir.getPath())).withValue(ConfigurationKeys.MR_JOB_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(mrJobRootDir.getPath())).withValue(ConfigurationKeys.SOURCE_CLASS_KEY, ConfigValueFactory.fromAnyRef(TestingSource.class.getName())).withValue(ConfigurationKeys.JOB_LOCK_ENABLED_KEY, ConfigValueFactory.fromAnyRef(false));
        JobSpec jobSpec2 = JobSpec.builder().withConfig(jobConf2).build();
        jled = (JobLauncherExecutionDriver) launcher.withJobLauncherType(JobLauncherFactory.JobLauncherType.MAPREDUCE).launchJob(jobSpec2);
        Assert.assertTrue(jled.getLegacyLauncher() instanceof MRJobLauncher);
        JobExecution jex2 = jled.getJobExecution();
        Assert.assertEquals(jex2.getJobSpecURI(), jobSpec2.getUri());
        Assert.assertEquals(jex2.getJobSpecVersion(), jobSpec2.getVersion());
        Assert.assertTrue(jex2.getLaunchTimeMillis() >= jex1.getLaunchTimeMillis());
    } finally {
        FileUtils.deleteDirectory(tmpTestDir);
    }
}