Examples with TestingEventBusAsserter org.apache.gobblin.writer.test.TestingEventBusAsserter used on opensource projects

Example 1 with TestingEventBusAsserter

use of org.apache.gobblin.writer.test.TestingEventBusAsserter in project incubator-gobblin by apache.

the class EmbeddedGobblinTest method testRunWithTemplate.

@Test
public void testRunWithTemplate() throws Exception {
    String eventBusId = this.getClass().getName();
    int numHellos = 5;
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/hello-world.template"));
    embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
    embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
    embeddedGobblin.setConfiguration(HelloWorldSource.NUM_HELLOS_FULL_KEY, Integer.toString(numHellos));
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= numHellos; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
}

Example 2 with TestingEventBusAsserter

use of org.apache.gobblin.writer.test.TestingEventBusAsserter in project incubator-gobblin by apache.

the class EmbeddedGobblinTest method testRunWithJobFile.

@Test
public void testRunWithJobFile() throws Exception {
    String eventBusId = this.getClass().getName() + ".jobFileTest";
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").jobFile(getClass().getResource("/testJobs/helloWorld.conf").getPath());
    embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName());
    embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId);
    JobExecutionResult result = embeddedGobblin.run();
    Assert.assertTrue(result.isSuccessful());
    ArrayList<String> expectedEvents = new ArrayList<>();
    for (int i = 1; i <= 10; ++i) {
        expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i));
    }
    asserter.assertNextValuesEq(expectedEvents);
    asserter.close();
}

Example 3 with TestingEventBusAsserter

use of org.apache.gobblin.writer.test.TestingEventBusAsserter in project incubator-gobblin by apache.

the class JobBrokerInjectionTest method testBrokerIsAcquiredAndShared.

@Test
public void testBrokerIsAcquiredAndShared() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testSubmitToJobCatalog");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("brokerTest/SimpleHelloWorldJob.jobconf")).build();
    final String eventBusId = js1.getConfig().resolve().getString(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY);
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    final StandardGobblinInstanceDriver instance = (StandardGobblinInstanceDriver) instanceLauncher.getDriver();
    final ArrayBlockingQueue<JobExecutionDriver> jobDrivers = new ArrayBlockingQueue<>(1);
    JobLifecycleListener js1Listener = new FilteredJobLifecycleListener(JobSpecFilter.eqJobSpecURI(js1.getUri()), new DefaultJobLifecycleListenerImpl(instance.getLog()) {

        @Override
        public void onJobLaunch(JobExecutionDriver jobDriver) {
            super.onJobLaunch(jobDriver);
            try {
                jobDrivers.offer(jobDriver, 5, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                instance.getLog().error("Offer interrupted.");
            }
        }
    });
    instance.registerWeakJobLifecycleListener(js1Listener);
    instance.getMutableJobCatalog().put(js1);
    JobExecutionDriver jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    JobExecutionResult jobResult = jobDriver.get(100000, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    Queue<TestingEventBuses.Event> events = asserter.getEvents();
    Set<Long> seenInstanceObjectIds = Sets.newHashSet();
    Set<Long> seenJobObjectIds = Sets.newHashSet();
    Set<Long> seenTaskObjectIds = Sets.newHashSet();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // Should see same instance and job id (only 1 id in the set), but 5 different task ids for each task
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 1);
    Assert.assertEquals(seenTaskObjectIds.size(), 5);
    asserter.clear();
    instance.getMutableJobCatalog().remove(js1.getUri());
    instance.getMutableJobCatalog().put(js1);
    jobDriver = jobDrivers.poll(10, TimeUnit.SECONDS);
    Assert.assertNotNull(jobDriver);
    jobResult = jobDriver.get(10, TimeUnit.SECONDS);
    Assert.assertTrue(jobResult.isSuccessful());
    events = asserter.getEvents();
    for (TestingEventBuses.Event event : events) {
        MyRecord record = (MyRecord) event.getValue();
        seenInstanceObjectIds.add(record.getInstanceSharedObjectId());
        seenJobObjectIds.add(record.getJobSharedObjectId());
        seenTaskObjectIds.add(record.getTaskSharedObjectId());
    }
    // A different job should produce a new shared object id
    Assert.assertEquals(seenInstanceObjectIds.size(), 1);
    Assert.assertEquals(seenJobObjectIds.size(), 2);
    Assert.assertEquals(seenTaskObjectIds.size(), 10);
}

Example 4 with TestingEventBusAsserter

use of org.apache.gobblin.writer.test.TestingEventBusAsserter in project incubator-gobblin by apache.

the class TextFileBasedSourceTest method test.

@Test(enabled = false, groups = { "disabledOnTravis" })
public void test() throws Exception {
    File stateStoreDir = Files.createTempDir();
    stateStoreDir.deleteOnExit();
    File dataDir = Files.createTempDir();
    dataDir.deleteOnExit();
    String eventBusId = UUID.randomUUID().toString();
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin gobblin = new EmbeddedGobblin().setTemplate("resource:///templates/textFileBasedSourceTest.template").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY, dataDir.getAbsolutePath()).setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStoreDir.getAbsolutePath()).setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId).setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true");
    Files.write("record1\nrecord2\nrecord3", new File(dataDir, "file1"), Charsets.UTF_8);
    Files.write("record4\nrecord5", new File(dataDir, "file2"), Charsets.UTF_8);
    gobblin.run();
    Set<Object> events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record1", "record2", "record3", "record4", "record5"));
    asserter.clear();
    // should only pull new files
    Files.write("record6\nrecord7", new File(dataDir, "file3"), Charsets.UTF_8);
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record6", "record7"));
    asserter.clear();
    // if we replace old file, it should repull that file
    Assert.assertTrue(new File(dataDir, "file2").delete());
    // Some systems don't provide modtime so gobblin can't keep of changed files.
    // run gobblin once with file2 deleted to update the snapshot
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertTrue(events.isEmpty());
    asserter.clear();
    Files.write("record8\nrecord9", new File(dataDir, "file2"), Charsets.UTF_8);
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record8", "record9"));
    asserter.clear();
}

Example 5 with TestingEventBusAsserter

use of org.apache.gobblin.writer.test.TestingEventBusAsserter in project incubator-gobblin by apache.

the class TextFileBasedSourceTest method testFileLimit.

@Test(enabled = false)
public void testFileLimit() throws Exception {
    File stateStoreDir = Files.createTempDir();
    stateStoreDir.deleteOnExit();
    File dataDir = Files.createTempDir();
    dataDir.deleteOnExit();
    String eventBusId = UUID.randomUUID().toString();
    TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId);
    EmbeddedGobblin gobblin = new EmbeddedGobblin().setTemplate("resource:///templates/textFileBasedSourceTest.template").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_DATA_DIRECTORY, dataDir.getAbsolutePath()).setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStoreDir.getAbsolutePath()).setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId).setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true").setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_MAX_FILES_PER_RUN, "2");
    Files.write("record1\nrecord2\nrecord3", new File(dataDir, "file1"), Charsets.UTF_8);
    Files.write("record4\nrecord5", new File(dataDir, "file2"), Charsets.UTF_8);
    Files.write("record6\nrecord7", new File(dataDir, "file3"), Charsets.UTF_8);
    gobblin.run();
    // should only pull first 2 files
    Set<Object> events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record1", "record2", "record3", "record4", "record5"));
    asserter.clear();
    gobblin.run();
    events = asserter.getEvents().stream().map(TestingEventBuses.Event::getValue).collect(Collectors.toSet());
    Assert.assertEquals(events, Sets.newHashSet("record6", "record7"));
    asserter.clear();
}