Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 41 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitToJobCatalog.

@Test
public /**
 * Test running of a job using the standard path of submitting to the job catalog
 */
void testSubmitToJobCatalog() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 42 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitWithTemplate.

@Test
public void testSubmitWithTemplate() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitWithTemplate");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseMap(ImmutableMap.of("numHellos", "5"))).withTemplate(new URI("resource:///gobblin/runtime/instance/SimpleHelloWorldJob.template")).build();
    ResolvedJobSpec js1Resolved = new ResolvedJobSpec(js1);
    final String eventBusId = js1Resolved.getConfig().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1Resolved.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 43 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class FSJobCatalogHelperTest method testloadGenericJobConfigs.

// This test doesn't delete framework attributes and
@Test(enabled = false)
public void testloadGenericJobConfigs() throws ConfigurationException, IOException, URISyntaxException {
    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, this.jobConfigDir.getAbsolutePath());
    List<JobSpec> jobSpecs = Lists.transform(Lists.newArrayList(loader.loadPullFilesRecursively(loader.getRootDirectory(), this.sysConfig, false)), this.converter);
    List<Properties> jobConfigs = convertJobSpecList2PropList(jobSpecs);
    Assert.assertEquals(jobConfigs.size(), 4);
    // test-job-conf-dir/test1/test11/test111.pull
    Properties jobProps1 = getJobConfigForFile(jobConfigs, "test111.pull");
    // 5 is consisting of three attributes, plus ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY
    // which is on purpose to keep
    // plus ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, which is not necessary to convert into JobSpec
    // but keep it here to avoid NullPointer exception and validation purpose for testing.
    Assert.assertEquals(jobProps1.stringPropertyNames().size(), 5);
    Assert.assertTrue(jobProps1.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY));
    Assert.assertEquals(jobProps1.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY), this.jobConfigDir.getAbsolutePath());
    Assert.assertEquals(jobProps1.getProperty("k1"), "d1");
    Assert.assertEquals(jobProps1.getProperty("k8"), "a8");
    Assert.assertEquals(jobProps1.getProperty("k9"), "a8");
    // test-job-conf-dir/test1/test11.pull
    Properties jobProps2 = getJobConfigForFile(jobConfigs, "test11.pull");
    Assert.assertEquals(jobProps2.stringPropertyNames().size(), 5);
    Assert.assertTrue(jobProps2.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY));
    Assert.assertEquals(jobProps2.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY), this.jobConfigDir.getAbsolutePath());
    Assert.assertEquals(jobProps2.getProperty("k1"), "c1");
    Assert.assertEquals(jobProps2.getProperty("k3"), "b3");
    Assert.assertEquals(jobProps2.getProperty("k6"), "a6");
    // test-job-conf-dir/test1/test12.PULL
    Properties jobProps3 = getJobConfigForFile(jobConfigs, "test12.PULL");
    Assert.assertEquals(jobProps3.stringPropertyNames().size(), 3);
    Assert.assertTrue(jobProps3.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY));
    Assert.assertEquals(jobProps3.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY), this.jobConfigDir.getAbsolutePath());
    Assert.assertEquals(jobProps3.getProperty("k7"), "a7");
    // test-job-conf-dir/test2/test21.PULL
    Properties jobProps4 = getJobConfigForFile(jobConfigs, "test21.PULL");
    Assert.assertEquals(jobProps4.stringPropertyNames().size(), 3);
    Assert.assertTrue(jobProps4.containsKey(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY));
    Assert.assertEquals(jobProps4.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY), this.jobConfigDir.getAbsolutePath());
    Assert.assertEquals(jobProps4.getProperty("k5"), "b5");
}

Example 44 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class TestJobCatalogListenersList method testCalls.

@Test
public void testCalls() {
    JobCatalogListenersList ll = new JobCatalogListenersList();
    JobSpec js1_1 = JobSpec.builder("test:job1").build();
    JobSpec js1_2 = JobSpec.builder("test:job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test:job2").build();
    JobCatalogListener l1 = Mockito.mock(JobCatalogListener.class);
    Mockito.doThrow(new RuntimeException("injected l1 failure")).when(l1).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion()));
    JobCatalogListener l2 = Mockito.mock(JobCatalogListener.class);
    Mockito.doThrow(new RuntimeException("injected l2 failure")).when(l2).onUpdateJob(Mockito.eq(js1_2));
    JobCatalogListener l3 = Mockito.mock(JobCatalogListener.class);
    Mockito.doThrow(new RuntimeException("injected l3 failure")).when(l3).onAddJob(Mockito.eq(js2));
    ll.addListener(l1);
    ll.addListener(l2);
    ll.addListener(l3);
    ll.onAddJob(js1_1);
    ll.onAddJob(js2);
    ll.onUpdateJob(js1_2);
    ll.onDeleteJob(js2.getUri(), js2.getVersion());
    Mockito.verify(l1).onAddJob(Mockito.eq(js1_1));
    Mockito.verify(l1).onAddJob(Mockito.eq(js2));
    Mockito.verify(l1).onUpdateJob(Mockito.eq(js1_2));
    Mockito.verify(l1).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion()));
    Mockito.verify(l2).onAddJob(Mockito.eq(js1_1));
    Mockito.verify(l2).onAddJob(Mockito.eq(js2));
    Mockito.verify(l2).onUpdateJob(Mockito.eq(js1_2));
    Mockito.verify(l2).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion()));
    Mockito.verify(l3).onAddJob(Mockito.eq(js1_1));
    Mockito.verify(l3).onAddJob(Mockito.eq(js2));
    Mockito.verify(l3).onUpdateJob(Mockito.eq(js1_2));
    Mockito.verify(l3).onDeleteJob(Mockito.eq(js2.getUri()), Mockito.eq(js2.getVersion()));
    Mockito.verifyNoMoreInteractions(l1, l2, l3);
}

Example 45 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class MockedKafkaJobMonitor method createMockCatalog.

private static MutableJobCatalog createMockCatalog(final Map<URI, JobSpec> jobSpecs) {
    MutableJobCatalog jobCatalog = Mockito.mock(MutableJobCatalog.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            JobSpec jobSpec = (JobSpec) invocation.getArguments()[0];
            jobSpecs.put(jobSpec.getUri(), jobSpec);
            return null;
        }
    }).when(jobCatalog).put(Mockito.any(JobSpec.class));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            URI uri = (URI) invocation.getArguments()[0];
            jobSpecs.remove(uri);
            return null;
        }
    }).when(jobCatalog).remove(Mockito.any(URI.class));
    return jobCatalog;
}