Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 16 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecProducer method convertToAvroJobSpec.

private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) {
    if (spec instanceof JobSpec) {
        JobSpec jobSpec = (JobSpec) spec;
        AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder();
        avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()).setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())).setMetadata(ImmutableMap.of(VERB_KEY, verb.name()));
        if (jobSpec.getTemplateURI().isPresent()) {
            avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString());
        }
        return avroJobSpecBuilder.build();
    } else {
        throw new RuntimeException("Unsupported spec type " + spec.getClass());
    }
}

Example 17 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}

Example 18 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestMutableCachingJobCatalog method test.

@Test
public void test() throws Exception {
    InMemoryJobCatalog baseCat = new InMemoryJobCatalog(Optional.<Logger>of(LoggerFactory.getLogger("baseCat")));
    baseCat.startAsync();
    baseCat.awaitRunning(2, TimeUnit.SECONDS);
    MutableCachingJobCatalog cachedCat = new MutableCachingJobCatalog(baseCat, Optional.<Logger>of(LoggerFactory.getLogger("cachedCat")));
    JobCatalogListener l = Mockito.mock(JobCatalogListener.class);
    cachedCat.addListener(l);
    cachedCat.startAsync();
    cachedCat.awaitRunning(10, TimeUnit.SECONDS);
    JobSpec js1_1 = JobSpec.builder("test:job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test:job1").withVersion("2").build();
    JobSpec js1_3 = JobSpec.builder("test:job1").withVersion("3").build();
    URI jsURI = new URI("test:job1");
    baseCat.put(js1_1);
    JobSpec res = cachedCat.getJobSpec(new URI("test:job1"));
    Assert.assertEquals(res, js1_1);
    baseCat.put(js1_2);
    res = cachedCat.getJobSpec(jsURI);
    Assert.assertEquals(res, js1_2);
    baseCat.remove(jsURI);
    try {
        cachedCat.getJobSpec(jsURI);
        Assert.fail("Expected JobSpecNotFoundException");
    } catch (JobSpecNotFoundException e) {
        Assert.assertEquals(e.getMissingJobSpecURI(), jsURI);
    }
    cachedCat.removeListener(l);
    cachedCat.put(js1_3);
    res = cachedCat.getJobSpec(jsURI);
    Assert.assertEquals(res, js1_3);
    res = baseCat.getJobSpec(jsURI);
    Assert.assertEquals(res, js1_3);
    cachedCat.remove(jsURI);
    try {
        cachedCat.getJobSpec(jsURI);
        Assert.fail("Expected JobSpecNotFoundException");
    } catch (JobSpecNotFoundException e) {
        Assert.assertEquals(e.getMissingJobSpecURI(), jsURI);
    }
    try {
        baseCat.getJobSpec(jsURI);
        Assert.fail("Expected JobSpecNotFoundException");
    } catch (JobSpecNotFoundException e) {
        Assert.assertEquals(e.getMissingJobSpecURI(), jsURI);
    }
    Mockito.verify(l).onAddJob(Mockito.eq(js1_1));
    Mockito.verify(l).onUpdateJob(Mockito.eq(js1_2));
    Mockito.verify(l).onDeleteJob(Mockito.eq(js1_2.getUri()), Mockito.eq(js1_2.getVersion()));
    Mockito.verifyNoMoreInteractions(l);
    cachedCat.stopAsync();
    cachedCat.awaitTerminated(10, TimeUnit.SECONDS);
    baseCat.stopAsync();
    baseCat.awaitTerminated(2, TimeUnit.SECONDS);
}

Example 19 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestNonObservingFSJobCatalog method testCallbacks.

@Test
public void testCallbacks() throws Exception {
    this.jobConfigDir = java.nio.file.Files.createTempDirectory(String.format("gobblin-test_%s_job-conf", this.getClass().getSimpleName())).toFile();
    this.jobConfigDirPath = new Path(this.jobConfigDir.getPath());
    try (PrintWriter printWriter = new PrintWriter(new Path(jobConfigDirPath, "job3.template").toString(), "UTF-8")) {
        printWriter.println("param1 = value1");
        printWriter.println("param2 = value2");
    }
    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, this.jobConfigDir.getPath());
    NonObservingFSJobCatalog cat = new NonObservingFSJobCatalog(ConfigUtils.propertiesToConfig(properties));
    cat.startAsync();
    cat.awaitRunning(10, TimeUnit.SECONDS);
    final Map<URI, JobSpec> specs = new Hashtable<>();
    JobCatalogListener l = Mockito.mock(JobCatalogListener.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onAddJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec spec = (JobSpec) invocation.getArguments()[0];
            specs.put(spec.getUri(), spec);
            return null;
        }
    }).when(l).onUpdateJob(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            URI uri = (URI) invocation.getArguments()[0];
            specs.remove(uri);
            return null;
        }
    }).when(l).onDeleteJob(Mockito.any(URI.class), Mockito.anyString());
    JobSpec js1_1 = JobSpec.builder("test_job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test_job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test_job2").withVersion("1").build();
    JobSpec js3 = JobSpec.builder("test_job3").withVersion("1").withTemplate(new URI("FS:///job3.template")).withConfig(ConfigBuilder.create().addPrimitive("job.template", "FS:///job3.template").build()).build();
    cat.addListener(l);
    cat.put(js1_1);
    Assert.assertTrue(specs.containsKey(js1_1.getUri()));
    JobSpec js1_1_notified = specs.get(js1_1.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_1_notified.getConfig(), js1_1.getConfig()));
    Assert.assertEquals(js1_1.getVersion(), js1_1_notified.getVersion());
    cat.put(js1_2);
    Assert.assertTrue(specs.containsKey(js1_2.getUri()));
    JobSpec js1_2_notified = specs.get(js1_2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js1_2_notified.getConfig(), js1_2.getConfig()));
    Assert.assertEquals(js1_2.getVersion(), js1_2_notified.getVersion());
    cat.put(js2);
    Assert.assertTrue(specs.containsKey(js2.getUri()));
    JobSpec js2_notified = specs.get(js2.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js2_notified.getConfig(), js2.getConfig()));
    Assert.assertEquals(js2.getVersion(), js2_notified.getVersion());
    cat.remove(js2.getUri());
    Assert.assertFalse(specs.containsKey(js2.getUri()));
    cat.put(js3);
    Assert.assertTrue(specs.containsKey(js3.getUri()));
    JobSpec js3_notified = specs.get(js3.getUri());
    Assert.assertTrue(ConfigUtils.verifySubset(js3_notified.getConfig(), js3.getConfig()));
    Assert.assertEquals(js3.getVersion(), js3_notified.getVersion());
    ResolvedJobSpec js3_resolved = new ResolvedJobSpec(js3_notified, cat);
    Assert.assertEquals(js3_resolved.getConfig().getString("param1"), "value1");
    Assert.assertEquals(js3_resolved.getConfig().getString("param2"), "value2");
    cat.stopAsync();
    cat.awaitTerminated(10, TimeUnit.SECONDS);
}

Example 20 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestJobLauncherExecutionDriver method testConstructor.

@Test
public void testConstructor() throws IOException, InterruptedException {
    File tmpTestDir = Files.createTempDir();
    try {
        File localJobRootDir = new File(tmpTestDir, "localJobRoot");
        Assert.assertTrue(localJobRootDir.mkdir());
        final Logger log = LoggerFactory.getLogger(getClass().getSimpleName() + ".testConstructor");
        Config jobConf1 = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob")).withValue(ConfigurationKeys.JOB_GROUP_KEY, ConfigValueFactory.fromAnyRef("myGroup")).withValue(ConfigurationKeys.JOB_DESCRIPTION_KEY, ConfigValueFactory.fromAnyRef("Awesome job")).withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(localJobRootDir.getPath())).withValue(ConfigurationKeys.SOURCE_CLASS_KEY, ConfigValueFactory.fromAnyRef(TestingSource.class.getName())).withValue(ConfigurationKeys.JOB_LOCK_ENABLED_KEY, ConfigValueFactory.fromAnyRef(false));
        JobSpec jobSpec1 = JobSpec.builder().withConfig(jobConf1).build();
        JobLauncherExecutionDriver.Launcher launcher = new JobLauncherExecutionDriver.Launcher().withJobLauncherType(JobLauncherFactory.JobLauncherType.LOCAL).withLog(log);
        JobLauncherExecutionDriver jled = (JobLauncherExecutionDriver) launcher.launchJob(jobSpec1);
        Assert.assertTrue(jled.getLegacyLauncher() instanceof LocalJobLauncher);
        JobExecution jex1 = jled.getJobExecution();
        Assert.assertEquals(jex1.getJobSpecURI(), jobSpec1.getUri());
        Assert.assertEquals(jex1.getJobSpecVersion(), jobSpec1.getVersion());
        Thread.sleep(5000);
        File mrJobRootDir = new File(tmpTestDir, "mrJobRoot");
        Assert.assertTrue(mrJobRootDir.mkdir());
        Config jobConf2 = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("myJob2")).withValue(ConfigurationKeys.JOB_GROUP_KEY, ConfigValueFactory.fromAnyRef("myGroup")).withValue(ConfigurationKeys.JOB_DESCRIPTION_KEY, ConfigValueFactory.fromAnyRef("Awesome job")).withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(mrJobRootDir.getPath())).withValue(ConfigurationKeys.MR_JOB_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(mrJobRootDir.getPath())).withValue(ConfigurationKeys.SOURCE_CLASS_KEY, ConfigValueFactory.fromAnyRef(TestingSource.class.getName())).withValue(ConfigurationKeys.JOB_LOCK_ENABLED_KEY, ConfigValueFactory.fromAnyRef(false));
        JobSpec jobSpec2 = JobSpec.builder().withConfig(jobConf2).build();
        jled = (JobLauncherExecutionDriver) launcher.withJobLauncherType(JobLauncherFactory.JobLauncherType.MAPREDUCE).launchJob(jobSpec2);
        Assert.assertTrue(jled.getLegacyLauncher() instanceof MRJobLauncher);
        JobExecution jex2 = jled.getJobExecution();
        Assert.assertEquals(jex2.getJobSpecURI(), jobSpec2.getUri());
        Assert.assertEquals(jex2.getJobSpecVersion(), jobSpec2.getVersion());
        Assert.assertTrue(jex2.getLaunchTimeMillis() >= jex1.getLaunchTimeMillis());
    } finally {
        FileUtils.deleteDirectory(tmpTestDir);
    }
}