Examples with AssertWithBackoff org.apache.gobblin.testing.AssertWithBackoff used on opensource projects

Example 1 with AssertWithBackoff

use of org.apache.gobblin.testing.AssertWithBackoff in project incubator-gobblin by apache.

the class ClusterIntegrationTest method waitForAndVerifyOutputFiles.

private void waitForAndVerifyOutputFiles() throws Exception {
    AssertWithBackoff asserter = AssertWithBackoff.create().logger(_logger).timeoutMs(60_000).maxSleepMs(100).backoffFactor(1.5);
    asserter.assertTrue(this::hasExpectedFilesBeenCreated, "Waiting for job-completion");
}

Example 2 with AssertWithBackoff

use of org.apache.gobblin.testing.AssertWithBackoff in project incubator-gobblin by apache.

the class GobblinClusterManagerTest method testSendShutdownRequest.

@Test
public void testSendShutdownRequest() throws Exception {
    Logger log = LoggerFactory.getLogger("testSendShutdownRequest");
    Closer closer = Closer.create();
    try {
        CuratorFramework curatorFramework = TestHelper.createZkClient(this.testingZKServer, closer);
        final GetInstanceMessageNumFunc getMessageNumFunc = new GetInstanceMessageNumFunc(GobblinClusterManagerTest.class.getSimpleName(), curatorFramework);
        AssertWithBackoff assertWithBackoff = AssertWithBackoff.create().logger(log).timeoutMs(30000);
        this.gobblinClusterManager.sendShutdownRequest();
        Assert.assertEquals(curatorFramework.checkExists().forPath(String.format("/%s/INSTANCES/%s/MESSAGES", GobblinClusterManagerTest.class.getSimpleName(), TestHelper.TEST_HELIX_INSTANCE_NAME)).getVersion(), 0);
        assertWithBackoff.assertEquals(getMessageNumFunc, 1, "1 message queued");
        // Give Helix sometime to handle the message
        assertWithBackoff.assertEquals(getMessageNumFunc, 0, "all messages processed");
    } finally {
        closer.close();
    }
}

Example 3 with AssertWithBackoff

use of org.apache.gobblin.testing.AssertWithBackoff in project incubator-gobblin by apache.

the class TestStandardGobblinInstanceLauncher method testDirectToExecutionDriver.

@Test
public /**
 * Test running of a job when submitted directly to the execution driver
 */
void testDirectToExecutionDriver() throws Exception {
    StandardGobblinInstanceLauncher.Builder instanceLauncherBuilder = StandardGobblinInstanceLauncher.builder().withInstanceName("testDirectToExecutionDriver");
    instanceLauncherBuilder.driver();
    StandardGobblinInstanceLauncher instanceLauncher = instanceLauncherBuilder.build();
    instanceLauncher.startAsync();
    instanceLauncher.awaitRunning(5, TimeUnit.SECONDS);
    JobSpec js1 = JobSpec.builder().withConfig(ConfigFactory.parseResources("gobblin/runtime/instance/SimpleHelloWorldJob.jobconf")).build();
    GobblinInstanceDriver instance = instanceLauncher.getDriver();
    final JobExecutionLauncher.StandardMetrics launcherMetrics = instance.getJobLauncher().getMetrics();
    AssertWithBackoff asb = new AssertWithBackoff().timeoutMs(100);
    checkLaunchJob(instanceLauncher, js1, instance);
    Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 1);
    Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 1);
    // Need to use assert with backoff because of race conditions with the callback that updates the
    // metrics
    asb.assertEquals(new Function<Void, Long>() {

        @Override
        public Long apply(Void input) {
            return launcherMetrics.getNumJobsCommitted().getCount();
        }
    }, 1l, "numJobsCommitted==1");
    Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
    Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
    checkLaunchJob(instanceLauncher, js1, instance);
    Assert.assertEquals(launcherMetrics.getNumJobsLaunched().getCount(), 2);
    Assert.assertEquals(launcherMetrics.getNumJobsCompleted().getCount(), 2);
    asb.assertEquals(new Function<Void, Long>() {

        @Override
        public Long apply(Void input) {
            return launcherMetrics.getNumJobsCommitted().getCount();
        }
    }, 2l, "numJobsCommitted==2");
    Assert.assertEquals(launcherMetrics.getNumJobsFailed().getCount(), 0);
    Assert.assertEquals(launcherMetrics.getNumJobsRunning().getValue().intValue(), 0);
}

Example 4 with AssertWithBackoff

use of org.apache.gobblin.testing.AssertWithBackoff in project incubator-gobblin by apache.

the class JobConfigFileMonitorTest method testAddNewJobConfigFile.

@Test(enabled = false)
public void testAddNewJobConfigFile() throws Exception {
    final Logger log = LoggerFactory.getLogger("testAddNewJobConfigFile");
    log.info("testAddNewJobConfigFile: start");
    AssertWithBackoff assertWithBackoff = AssertWithBackoff.create().logger(log).timeoutMs(15000);
    assertWithBackoff.assertEquals(new GetNumScheduledJobs(), 3, "3 scheduled jobs");
    /* Set a time gap, to let the monitor recognize the "3-file" status as old status,
    so that new added file can be discovered */
    Thread.sleep(1000);
    // Create a new job configuration file by making a copy of an existing
    // one and giving a different job name
    Properties jobProps = new Properties();
    jobProps.load(new FileReader(new File(this.jobConfigDir, "GobblinTest1.pull")));
    jobProps.setProperty(ConfigurationKeys.JOB_NAME_KEY, "Gobblin-test-new");
    this.newJobConfigFile = new File(this.jobConfigDir, "Gobblin-test-new.pull");
    jobProps.store(new FileWriter(this.newJobConfigFile), null);
    assertWithBackoff.assertEquals(new GetNumScheduledJobs(), 4, "4 scheduled jobs");
    Set<String> jobNames = Sets.newHashSet(this.jobScheduler.getScheduledJobs());
    Set<String> expectedJobNames = ImmutableSet.<String>builder().add("GobblinTest1", "GobblinTest2", "GobblinTest3", "Gobblin-test-new").build();
    Assert.assertEquals(jobNames, expectedJobNames);
    log.info("testAddNewJobConfigFile: end");
}

Example 5 with AssertWithBackoff

use of org.apache.gobblin.testing.AssertWithBackoff in project incubator-gobblin by apache.

the class TestDefaultGobblinInstanceDriverImpl method testScheduling.

@Test
public void testScheduling() throws Exception {
    final Logger log = LoggerFactory.getLogger(getClass().getName() + ".testScheduling");
    final Optional<Logger> loggerOpt = Optional.of(log);
    InMemoryJobCatalog jobCatalog = new InMemoryJobCatalog(loggerOpt);
    JobSpecScheduler scheduler = Mockito.mock(JobSpecScheduler.class);
    JobExecutionLauncher jobLauncher = Mockito.mock(JobExecutionLauncher.class);
    Configurable sysConfig = DefaultConfigurableImpl.createFromConfig(ConfigFactory.empty());
    final DefaultGobblinInstanceDriverImpl driver = new StandardGobblinInstanceDriver("testScheduling", sysConfig, jobCatalog, scheduler, jobLauncher, Optional.<MetricContext>absent(), loggerOpt, Collections.<GobblinInstancePluginFactory>emptyList(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()));
    JobSpec js1_1 = JobSpec.builder("test.job1").withVersion("1").build();
    JobSpec js1_2 = JobSpec.builder("test.job1").withVersion("2").build();
    JobSpec js2 = JobSpec.builder("test.job2").withVersion("1").build();
    driver.startAsync().awaitRunning(1000, TimeUnit.MILLISECONDS);
    long startTimeMs = System.currentTimeMillis();
    Assert.assertTrue(driver.isRunning());
    Assert.assertTrue(driver.isInstrumentationEnabled());
    Assert.assertNotNull(driver.getMetricContext());
    jobCatalog.put(js1_1);
    AssertWithBackoff awb = AssertWithBackoff.create().backoffFactor(1.5).maxSleepMs(100).timeoutMs(1000).logger(log);
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            log.debug("upFlag=" + driver.getMetrics().getUpFlag().getValue().intValue());
            return driver.getMetrics().getUpFlag().getValue().intValue() == 1;
        }
    }, "upFlag==1");
    jobCatalog.put(js2);
    jobCatalog.put(js1_2);
    jobCatalog.remove(js2.getUri());
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js1_1), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js1_1, null));
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js2, null));
    Mockito.when(scheduler.scheduleJob(Mockito.eq(js1_2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class))).thenReturn(DefaultJobSpecScheduleImpl.createNoSchedule(js1_2, null));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js1_1), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).scheduleJob(Mockito.eq(js1_2), Mockito.any(DefaultGobblinInstanceDriverImpl.JobSpecRunnable.class));
    Mockito.verify(scheduler).unscheduleJob(Mockito.eq(js2.getUri()));
    final long elapsedMs = System.currentTimeMillis() - startTimeMs;
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            long uptimeMs = driver.getMetrics().getUptimeMs().getValue().longValue();
            return uptimeMs >= elapsedMs;
        }
    }, "uptime > elapsedMs");
    long uptimeMs = driver.getMetrics().getUptimeMs().getValue().longValue();
    Assert.assertTrue(uptimeMs <= 2 * elapsedMs, "uptime=" + uptimeMs + " elapsedMs=" + elapsedMs);
    driver.stopAsync();
    driver.awaitTerminated(100, TimeUnit.MILLISECONDS);
    Assert.assertEquals(driver.state(), State.TERMINATED);
    Assert.assertEquals(driver.getMetrics().getUpFlag().getValue().intValue(), 0);
    // Need an assert with retries because Guava service container notifications are async
    awb.assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            log.debug("upTimeMs=" + driver.getMetrics().getUptimeMs().getValue().longValue());
            return driver.getMetrics().getUptimeMs().getValue().longValue() == 0;
        }
    }, "upTimeMs==0");
}