Examples with EmbeddedGobblin org.apache.gobblin.runtime.embedded.EmbeddedGobblin used on opensource projects

Example 1 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testHiveRegistrationFailure.

@Test
public void testHiveRegistrationFailure() throws Exception {
    File basePath = Files.createTempDir();
    basePath.deleteOnExit();
    GenericRecord r1 = createRandomRecord();
    // success dataset
    String path1 = TestCompactionSuiteFactories.DATASET_SUCCESS + "/20_30/run_2017-04-03-10-20";
    File jobDir1 = new File(basePath, path1);
    Assert.assertTrue(jobDir1.mkdirs());
    writeFileWithContent(jobDir1, "file_random", r1, 20);
    // failed dataset
    String path2 = TestCompactionSuiteFactories.DATASET_FAIL + "/20_30/run_2017-04-03-10-20";
    File jobDir2 = new File(basePath, path2);
    Assert.assertTrue(jobDir2.mkdirs());
    writeFileWithContent(jobDir2, "file_random", r1, 20);
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblinForHiveRegistrationFailure("hive_registration_failure", basePath.getAbsolutePath().toString());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertFalse(result.isSuccessful());
}

Example 2 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testPrioritization.

@Test
public void testPrioritization() throws Exception {
    File basePath = Files.createTempDir();
    basePath.deleteOnExit();
    GenericRecord r1 = createRandomRecord();
    // verify 24 hours
    for (int i = 1; i < 3; ++i) {
        String path = "Identity/MemberAccount/minutely/2017/04/03/" + i + "/20_30/run_2017-04-03-10-20";
        File jobDir = new File(basePath, path);
        Assert.assertTrue(jobDir.mkdirs());
        writeFileWithContent(jobDir, "file_random", r1, 20);
    }
    for (int i = 1; i < 3; ++i) {
        String path = "EVG/People/minutely/2017/04/03/" + i + "/20_30/run_2017-04-03-10-20";
        File jobDir = new File(basePath, path);
        Assert.assertTrue(jobDir.mkdirs());
        writeFileWithContent(jobDir, "file_random", r1, 20);
    }
    for (int i = 1; i < 3; ++i) {
        String path = "BizProfile/BizCompany/minutely/2017/04/03/" + i + "/20_30/run_2017-04-03-10-20";
        File jobDir = new File(basePath, path);
        Assert.assertTrue(jobDir.mkdirs());
        writeFileWithContent(jobDir, "file_random", r1, 20);
    }
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblinWithPriority("workunit_stream_priority", basePath.getAbsolutePath().toString());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
}

Example 3 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testWorkUnitStreamForAllFailures.

@Test
public void testWorkUnitStreamForAllFailures() throws Exception {
    File basePath = Files.createTempDir();
    basePath.deleteOnExit();
    GenericRecord r1 = createRandomRecord();
    // verify 24 hours
    for (int i = 1; i < 24; ++i) {
        String path = "Identity/MemberAccount/minutely/2017/04/03/" + i + "/20_30/run_2017-04-03-10-20";
        File jobDir = new File(basePath, path);
        Assert.assertTrue(jobDir.mkdirs());
        writeFileWithContent(jobDir, "file_random", r1, 20);
    }
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblinForAllFailures("workunit_stream_all_failure", basePath.getAbsolutePath().toString());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertFalse(result.isSuccessful());
}

Example 4 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class MRCompactionTaskTest method testWorkUnitStream.

@Test
public void testWorkUnitStream() throws Exception {
    File basePath = Files.createTempDir();
    basePath.deleteOnExit();
    GenericRecord r1 = createRandomRecord();
    // verify 24 hours
    for (int i = 22; i < 24; ++i) {
        String path = "Identity/MemberAccount/minutely/2017/04/03/" + i + "/20_30/run_2017-04-03-10-20";
        File jobDir = new File(basePath, path);
        Assert.assertTrue(jobDir.mkdirs());
        writeFileWithContent(jobDir, "file_random", r1, 20);
    }
    EmbeddedGobblin embeddedGobblin = createEmbeddedGobblin("workunit_stream", basePath.getAbsolutePath().toString());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
}

Example 5 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class TestWorkUnitStreamSource method test.

/**
 * This test uses a slow source to verify that we can stream work units through local job launcher, with available units
 * being processes eagerly even if not all work units are available.
 */
@Test
public void test() throws Exception {
    String eventBusId = UUID.randomUUID().toString();
    MyListener listener = new MyListener();
    EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
    eventBus.register(listener);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("testStreamedSource").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, MySource.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "5");
    JobExecutionDriver driver = embeddedGobblin.runAsync();
    if (!listener.iteratorReady.tryAcquire(2, TimeUnit.SECONDS)) {
        throw new RuntimeException("Failed to get start signal.");
    }
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    Assert.assertTrue(listener.tasksRun.tryAcquire(500, TimeUnit.MILLISECONDS));
    Assert.assertFalse(listener.tasksRun.tryAcquire(50, TimeUnit.MILLISECONDS));
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    eventBus.post(new MySource.NextWorkUnit());
    JobExecutionResult result = driver.get(5, TimeUnit.SECONDS);
    Assert.assertTrue(result.isSuccessful());
    SetMultimap<String, Integer> eventsSeen = listener.getEventsSeenMap();
    Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.RUN_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.COMMIT_EVENT), expected);
    Assert.assertEquals(eventsSeen.get(EventBusPublishingTaskFactory.PUBLISH_EVENT), expected);
}