Examples with JobExecutionResult org.apache.gobblin.runtime.api.JobExecutionResult used on opensource projects

Example 16 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class MRTaskFactoryTest method test.

@Test
public void test() throws Exception {
    File inputSuperPath = Files.createTempDir();
    inputSuperPath.deleteOnExit();
    File outputSuperPath = Files.createTempDir();
    outputSuperPath.deleteOnExit();
    File job1Dir = new File(inputSuperPath, "job1");
    Assert.assertTrue(job1Dir.mkdir());
    writeFileWithContent(job1Dir, "file1", "word1 word1 word2");
    writeFileWithContent(job1Dir, "file2", "word2 word2 word2");
    File job2Dir = new File(inputSuperPath, "job2");
    Assert.assertTrue(job2Dir.mkdir());
    writeFileWithContent(job2Dir, "file1", "word1 word2 word2");
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("WordCounter").setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MRWordCountSource.class.getName()).setConfiguration(MRWordCountSource.INPUT_DIRECTORIES_KEY, job1Dir.getAbsolutePath() + "," + job2Dir.getAbsolutePath()).setConfiguration(MRWordCountSource.OUTPUT_LOCATION, outputSuperPath.getAbsolutePath());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    File output1 = new File(new File(outputSuperPath, "job1"), "part-r-00000");
    Assert.assertTrue(output1.exists());
    Map<String, Integer> counts = parseCounts(output1);
    Assert.assertEquals((int) counts.get("word1"), 2);
    Assert.assertEquals((int) counts.get("word2"), 4);
    File output2 = new File(new File(outputSuperPath, "job2"), "part-r-00000");
    Assert.assertTrue(output2.exists());
    counts = parseCounts(output2);
    Assert.assertEquals((int) counts.get("word1"), 1);
    Assert.assertEquals((int) counts.get("word2"), 2);
}

Example 17 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitToJobCatalog.

@Test
public /**
 * Test running of a job using the standard path of submitting to the job catalog
 */
void testSubmitToJobCatalog() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 18 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testSubmitWithTemplate.

@Test
public void testSubmitWithTemplate() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitWithTemplate");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseMap(ImmutableMap.of("numHellos", "5"))).withTemplate(new URI("resource:///gobblin/runtime/instance/SimpleHelloWorldJob.template")).build();
    ResolvedJobSpec js1Resolved = new ResolvedJobSpec(js1);
    final String eventBusId = js1Resolved.getConfig().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    instanceLauncher.stopAsync();
    final int numHellos = js1Resolved.getConfig().getInt(HelloWorldSource.NUM_HELLOS_FULL_KEY);
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
    instanceLauncher.awaitTerminated(5, TimeUnit.SECONDS);
}

Example 19 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testNonDedup.

@Test
public void testNonDedup() throws Exception {
    File basePath = Files.createTempDir();
    basePath.deleteOnExit();
    File jobDir = new File(basePath, "Identity/MemberAccount/minutely/2017/04/03/10/20_30/run_2017-04-03-10-20");
    Assert.assertTrue(jobDir.mkdirs());
    GenericRecord r1 = createRandomRecord();
    GenericRecord r2 = createRandomRecord();
    writeFileWithContent(jobDir, "file1", r1, 20);
    writeFileWithContent(jobDir, "file2", r2, 18);
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblin("non-dedup", basePath.getAbsolutePath().toString());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
}

Example 20 with JobExecutionResult

use of org.apache.gobblin.runtime.api.JobExecutionResult in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testRecompaction.

@Test
public void testRecompaction() throws Exception {
    FileSystem fs = getFileSystem();
    String basePath = "/tmp/testRecompaction";
    fs.delete(new Path(basePath), true);
    File jobDir = new File(basePath, "Identity/MemberAccount/minutely/2017/04/03/10/20_30/run_2017-04-03-10-20");
    Assert.assertTrue(jobDir.mkdirs());
    GenericRecord r1 = createRandomRecord();
    writeFileWithContent(jobDir, "file1", r1, 20);
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblin("Recompaction-First", basePath);
    JobExecutionResult result = embeddedGobblin.run();
    long recordCount = InputRecordCountHelper.readRecordCount(fs, (new Path(basePath, new Path("Identity/MemberAccount/hourly/2017/04/03/10"))));
    Assert.assertTrue(result.isSuccessful());
    Assert.assertEquals(recordCount, 20);
    // Now write more avro files to input dir
    writeFileWithContent(jobDir, "file2", r1, 22);
    EmbeddedGobblin embeddedGobblin_2 = createEmbeddedGobblin("Recompaction-Second", basePath);
    embeddedGobblin_2.run();
    Assert.assertTrue(result.isSuccessful());
    // If recompaction is succeeded, a new record count should be written.
    recordCount = InputRecordCountHelper.readRecordCount(fs, (new Path(basePath, new Path("Identity/MemberAccount/hourly/2017/04/03/10"))));
    Assert.assertEquals(recordCount, 42);
    Assert.assertTrue(fs.exists(new Path(basePath, "Identity/MemberAccount/hourly/2017/04/03/10")));
}