Examples with SpecCompiler org.apache.gobblin.service.modules.flow.SpecCompiler used on opensource projects

Example 1 with SpecCompiler

use of org.apache.gobblin.service.modules.flow.SpecCompiler in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerUnschedule.

/**
 * Test that flowSpecs that throw compilation errors do not block the scheduling of other flowSpecs
 */
@Test
public void testJobSchedulerUnschedule() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    // Assume that the catalog can store corrupted flows
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"));
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    FlowSpec flowSpec2 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec2"));
    // Ensure that these flows are scheduled
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    flowCatalog.put(flowSpec2, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 3);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    SchedulerService schedulerService = new SchedulerService(new Properties());
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService);
    schedulerService.startAsync().awaitRunning();
    scheduler.startUp();
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 3) {
                return scheduler.scheduledFlowSpecs.containsKey("spec0") && scheduler.scheduledFlowSpecs.containsKey("spec1") && scheduler.scheduledFlowSpecs.containsKey("spec2");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
    // set scheduler to be inactive and unschedule flows
    scheduler.setActive(false);
    Collection<Invocation> invocations = Mockito.mockingDetails(mockOrchestrator).getInvocations();
    for (Invocation invocation : invocations) {
        // ensure that orchestrator is not calling remove
        Assert.assertFalse(invocation.getMethod().getName().equals("remove"));
    }
    Assert.assertEquals(scheduler.scheduledFlowSpecs.size(), 0);
    Assert.assertEquals(schedulerService.getScheduler().getJobGroupNames().size(), 0);
}

Example 2 with SpecCompiler

use of org.apache.gobblin.service.modules.flow.SpecCompiler in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerInitWithFailedSpec.

/**
 * Test that flowSpecs that throw compilation errors do not block the scheduling of other flowSpecs
 */
@Test
public void testJobSchedulerInitWithFailedSpec() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    // Assume that the catalog can store corrupted flows
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"), MockedSpecCompiler.UNCOMPILABLE_FLOW);
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    FlowSpec flowSpec2 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec2"));
    // Ensure that these flows are scheduled
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    flowCatalog.put(flowSpec2, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 3);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, null);
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 2) {
                return scheduler.scheduledFlowSpecs.containsKey("spec1") && scheduler.scheduledFlowSpecs.containsKey("spec2");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
}

Example 3 with SpecCompiler

use of org.apache.gobblin.service.modules.flow.SpecCompiler in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerInit.

/**
 * Test whenever JobScheduler is calling setActive, the FlowSpec is loading into scheduledFlowSpecs (eventually)
 */
@Test
public void testJobSchedulerInit() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"));
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 2);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, null);
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 2) {
                return scheduler.scheduledFlowSpecs.containsKey("spec0") && scheduler.scheduledFlowSpecs.containsKey("spec1");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
}