Examples with EmbeddedGobblin org.apache.gobblin.runtime.embedded.EmbeddedGobblin used on opensource projects

Example 6 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class CustomTaskTest method testCustomTask.

@Test
public void testCustomTask() throws Exception {
    String eventBusId = UUID.randomUUID().toString();
    EventBusPublishingTaskFactory.EventListener listener = new EventBusPublishingTaskFactory.EventListener();
    EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
    eventBus.register(listener);
    JobExecutionResult result = new EmbeddedGobblin("testJob").setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, EventBusPublishingTaskFactory.Source.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "10").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).run();
    Assert.assertTrue(result.isSuccessful());
    SetMultimap<String, Integer> seenEvents = HashMultimap.create();
    Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
    for (EventBusPublishingTaskFactory.Event event : listener.getEvents()) {
        seenEvents.put(event.getType(), event.getId());
    }
    Assert.assertEquals(seenEvents.get("run"), expected);
    Assert.assertEquals(seenEvents.get("commit"), expected);
    Assert.assertEquals(seenEvents.get("publish"), expected);
}

Example 7 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class CustomTaskTest method testTaskFailsWithException.

@Test(timeOut = 30000)
public void testTaskFailsWithException() throws Exception {
    // Test that the job runner fails with a reasonable amount of time if a custom task throws an exception
    JobExecutionResult result = new EmbeddedGobblin("alwaysThrowsJob").setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, FailsWithExceptionTaskFactory.Source.class.getName()).run();
    Assert.assertFalse(result.isSuccessful());
}

Example 8 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class TextFileBasedSourceTest method test.

@Test(enabled = false, groups = { "disabledOnTravis" })
public void test() throws Exception {
    File stateStoreDir = Files.createTempDir();
    stateStoreDir.deleteOnExit();
    File dataDir = Files.createTempDir();
    dataDir.deleteOnExit();
    String eventBusId = UUID.randomUUID().toString();
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin gobblin = new EmbeddedGobblin().setTemplate("resource:///templates/textFileBasedSourceTest.template").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY, dataDir.getAbsolutePath()).setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStoreDir.getAbsolutePath()).setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId).setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true");
    Files.write("record1\nrecord2\nrecord3", new File(dataDir, "file1"), Charsets.UTF_8);
    Files.write("record4\nrecord5", new File(dataDir, "file2"), Charsets.UTF_8);
    gobblin.run();
    Set<Object> events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record1", "record2", "record3", "record4", "record5"));
    asserter.clear();
    // should only pull new files
    Files.write("record6\nrecord7", new File(dataDir, "file3"), Charsets.UTF_8);
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record6", "record7"));
    asserter.clear();
    // if we replace old file, it should repull that file
    Assert.assertTrue(new File(dataDir, "file2").delete());
    // Some systems don't provide modtime so gobblin can't keep of changed files.
    // run gobblin once with file2 deleted to update the snapshot
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertTrue(events.isEmpty());
    asserter.clear();
    Files.write("record8\nrecord9", new File(dataDir, "file2"), Charsets.UTF_8);
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record8", "record9"));
    asserter.clear();
}

Example 9 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class TextFileBasedSourceTest method testFileLimit.

@Test(enabled = false)
public void testFileLimit() throws Exception {
    File stateStoreDir = Files.createTempDir();
    stateStoreDir.deleteOnExit();
    File dataDir = Files.createTempDir();
    dataDir.deleteOnExit();
    String eventBusId = UUID.randomUUID().toString();
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin gobblin = new EmbeddedGobblin().setTemplate("resource:///templates/textFileBasedSourceTest.template").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY, dataDir.getAbsolutePath()).setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStoreDir.getAbsolutePath()).setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId).setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_MAX_FILES_PER_RUN, "2");
    Files.write("record1\nrecord2\nrecord3", new File(dataDir, "file1"), Charsets.UTF_8);
    Files.write("record4\nrecord5", new File(dataDir, "file2"), Charsets.UTF_8);
    Files.write("record6\nrecord7", new File(dataDir, "file3"), Charsets.UTF_8);
    gobblin.run();
    // should only pull first 2 files
    Set<Object> events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record1", "record2", "record3", "record4", "record5"));
    asserter.clear();
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record6", "record7"));
    asserter.clear();
}

Example 10 with EmbeddedGobblin

use of org.apache.gobblin.runtime.embedded.EmbeddedGobblin in project incubator-gobblin by apache.

the class CustomTaskTest method testStatePersistence.

@Test
public void testStatePersistence() throws Exception {
    File stateStore = Files.createTempDir();
    stateStore.deleteOnExit();
    String eventBusId = UUID.randomUUID().toString();
    EventBusPublishingTaskFactory.EventListener listener = new EventBusPublishingTaskFactory.EventListener();
    EventBus eventBus = TestingEventBuses.getEventBus(eventBusId);
    eventBus.register(listener);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("testJob").setConfiguration(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY, eventBusId).setConfiguration(ConfigurationKeys.SOURCE_CLASS_KEY, EventBusPublishingTaskFactory.Source.class.getName()).setConfiguration(EventBusPublishingTaskFactory.Source.NUM_TASKS_KEY, "2").setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, Boolean.toString(true)).setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStore.getAbsolutePath());
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    SetMultimap<String, Integer> seenEvents = HashMultimap.create();
    for (EventBusPublishingTaskFactory.Event event : listener.getEvents()) {
        seenEvents.put(event.getType(), event.getId());
    }
    Assert.assertEquals(seenEvents.get("previousState").size(), 0);
    result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    seenEvents = HashMultimap.create();
    for (EventBusPublishingTaskFactory.Event event : listener.getEvents()) {
        seenEvents.put(event.getType(), event.getId());
    }
    Assert.assertEquals(seenEvents.get("previousState"), Sets.newHashSet(0, 1));
}